diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/.github/workflows/ci.yml b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/.github/workflows/ci.yml new file mode 100644 index 00000000..9c7ef790 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/.github/workflows/ci.yml @@ -0,0 +1,46 @@ +# ©AngelaMos | 2026 +# ci.yml + +name: ci + +on: + push: + paths: + - "PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/**" + pull_request: + paths: + - "PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/**" + +defaults: + run: + working-directory: PROJECTS/intermediate/ja3-ja4-tls-fingerprinting + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + rust: [stable, "1.85"] + steps: + - uses: actions/checkout@v4 + - name: Install libpcap + run: sudo apt-get update && sudo apt-get install -y libpcap-dev + - name: Install Rust ${{ matrix.rust }} + run: rustup toolchain install ${{ matrix.rust }} --component clippy rustfmt --profile minimal + - name: Build + run: cargo +${{ matrix.rust }} build --workspace --all-targets + - name: Test + run: cargo +${{ matrix.rust }} test --workspace + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install libpcap + run: sudo apt-get update && sudo apt-get install -y libpcap-dev + - name: Install Rust + run: rustup toolchain install stable --component clippy rustfmt --profile minimal + - name: Format + run: cargo fmt --all --check + - name: Clippy + run: cargo clippy --workspace --all-targets -- -D warnings diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/.gitignore b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/.gitignore new file mode 100644 index 00000000..50b47180 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/.gitignore @@ -0,0 +1,9 @@ +# ©AngelaMos | 2026 +# .gitignore + +docs/ +target/ +*.db +*.sqlite +fuzz/corpus/ +fuzz/artifacts/ diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/Cargo.lock b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/Cargo.lock new file mode 100644 index 00000000..1759a05a --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/Cargo.lock @@ -0,0 +1,1985 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "circular" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fc239e0f6cb375d2402d48afb92f76f5404fd1df208a41930ec81eda078bea" + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "typenum", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "etherparse" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a48e7bdc36cdf86876a6890c0840957029374ea07f6697c96fa15898730375" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "spin", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "pcap" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2eecc2ddc671ec563b5b39f846556aade68a65d1afb14d8fe6b30b0457d75" +dependencies = [ + "bitflags 1.3.2", + "errno 0.2.8", + "libc", + "libloading", + "pkg-config", + "regex", + "windows-sys 0.36.1", +] + +[[package]] +name = "pcap-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f8d57cc6bdf76d7abd6d3cc1113278047dab29c2ff6d97190e8d1c29d4efdac" +dependencies = [ + "circular", + "nom", + "rusticata-macros", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.13.0", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags 2.13.0", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.0", + "errno 0.3.14", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno 0.3.14", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tlsfp" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "clap", + "etherparse", + "flume", + "pcap", + "pcap-parser", + "rusqlite", + "serde", + "serde_json", + "tlsfp-core", + "tokio", + "tower", + "tower-http", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tlsfp-core" +version = "0.1.0" +dependencies = [ + "aes", + "aes-gcm", + "ctr", + "etherparse", + "hex", + "hkdf", + "md-5", + "pcap-parser", + "proptest", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror", +] + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.0", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen 0.46.0", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.13.0", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.13.0", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/Cargo.toml b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/Cargo.toml new file mode 100644 index 00000000..5cf2cdc8 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/Cargo.toml @@ -0,0 +1,66 @@ +# ©AngelaMos | 2026 +# Cargo.toml + +[workspace] +resolver = "3" +members = ["crates/tlsfp-core", "crates/tlsfp"] + +[workspace.package] +version = "0.1.0" +edition = "2024" +rust-version = "1.85" +license = "MIT" +authors = ["Carter Perez"] +repository = "https://github.com/CarterPerez-dev/Cybersecurity-Projects" + +[workspace.dependencies] +tlsfp-core = { path = "crates/tlsfp-core" } + +winnow = "0.7" +md-5 = "0.10" +sha2 = "0.10" +hex = "0.4" +smallvec = "1" +thiserror = "2" +anyhow = "1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +etherparse = "0.18" +pcap-parser = "0.16" +pcap = "2" + +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal"] } +flume = "0.11" + +rusqlite = { version = "0.32", features = ["bundled"] } + +axum = "0.8" +tower-http = { version = "0.6", features = ["fs", "trace", "cors"] } +tower = "0.5" + +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +clap = { version = "4", features = ["derive"] } + +hkdf = "0.12" +aes = "0.8" +aes-gcm = "0.10" +ctr = "0.9" + +[workspace.lints.rust] +unsafe_code = "forbid" + +[workspace.lints.clippy] +pedantic = { level = "warn", priority = -1 } +module_name_repetitions = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +must_use_candidate = "allow" +doc_markdown = "allow" + +[profile.release] +opt-level = 3 +lto = "thin" +codegen-units = 1 diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/LICENSE b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/LICENSE new file mode 100644 index 00000000..a615e738 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Carter Perez + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/NOTICE.md b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/NOTICE.md new file mode 100644 index 00000000..8947fe21 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/NOTICE.md @@ -0,0 +1,61 @@ +# Third-party fingerprint algorithm licensing + +This project implements several published TLS fingerprinting algorithms. They do +not all carry the same license, and the split matters. This notice records what +applies to what. + +## Our own code + +Everything under `crates/` is original work licensed under the MIT License (see +`LICENSE`). No source code from any reference implementation was copied. The +algorithms were implemented from their published specifications. + +## JA3 and JA3S + +JA3 and JA3S were created by John Althouse, Jeff Atkinson, and Josh Atkins at +Salesforce and released under the BSD 3-Clause license at +`https://github.com/salesforce/ja3`. The algorithm is free to implement and use. +That repository was archived in May 2025 and is no longer maintained. + +## JA4 (TLS client fingerprint) + +JA4, the TLS client fingerprint, is licensed by FoxIO under the BSD 3-Clause +license, separately from the rest of the JA4+ suite. FoxIO has stated it holds +no patents on JA4 TLS client fingerprinting. The canonical license text is at +`https://github.com/FoxIO-LLC/ja4/blob/main/LICENSE-JA4`. JA4 may be implemented +and used without restriction. + +## JA4S, JA4H, JA4X, JA4T (the rest of the JA4+ suite) + +The remaining JA4+ fingerprints implemented here are licensed under the FoxIO +License 1.1, and the methods are patent pending. The canonical license text is +at `https://github.com/FoxIO-LLC/ja4/blob/main/LICENSE`. + +The FoxIO License 1.1 grants the right to use and modify these methods for non +commercial purposes, which it defines to include personal use, academic research +and development, and internal evaluation. It excludes any use for which a fee is +charged and excludes providing the methods as a hosted or managed service to +others. + +This project is a free, open source, educational tool. It is not sold, not +monetized, and not offered as a service. It therefore falls within the non +commercial grant of the FoxIO License 1.1. Anyone who forks this project and +intends to monetize it, or to offer it as a hosted service, must obtain an OEM +license from FoxIO for the JA4+ methods first. + +This is the same boundary that led the Suricata project to ship only JA4 and not +the rest of the JA4+ suite: their GPL licensing is incompatible with the FoxIO +License 1.1. This project is MIT licensed and non commercial, so both the GPL +incompatibility and the monetization restriction are avoided. + +## Seeded fingerprint data + +The intelligence database is seeded from public sources with their own terms: + +- abuse.ch SSLBL JA3 feed is released under CC0 and a snapshot may be + redistributed. Its known limitation, that the fingerprints have not been + tested against benign traffic and may produce false positives, is surfaced in + the tool output. +- The salesforce/ja3 application lists are MIT licensed. +- The ja4db.com database has no stated redistribution license, so it is fetched + at install time rather than bundled, and entries are validated on import. diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/Cargo.toml b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/Cargo.toml new file mode 100644 index 00000000..63237bba --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/Cargo.toml @@ -0,0 +1,34 @@ +# ©AngelaMos | 2026 +# Cargo.toml + +[package] +name = "tlsfp-core" +description = "TLS handshake parsing and JA3/JA4 fingerprint computation" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +md-5.workspace = true +sha2.workspace = true +hex.workspace = true +smallvec.workspace = true +thiserror.workspace = true +serde.workspace = true +etherparse.workspace = true +pcap-parser.workspace = true +hkdf.workspace = true +aes.workspace = true +aes-gcm.workspace = true +ctr.workspace = true + +[dev-dependencies] +hex.workspace = true +proptest = "1" +serde_json.workspace = true diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/der.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/der.rs new file mode 100644 index 00000000..7041b881 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/der.rs @@ -0,0 +1,110 @@ +// ©AngelaMos | 2026 +// der.rs + +use crate::error::{ParseError, Result}; + +/// DER tag bytes that the certificate walk needs to recognize. +pub mod tag { + pub const OBJECT_IDENTIFIER: u8 = 0x06; + pub const SEQUENCE: u8 = 0x30; + pub const SET: u8 = 0x31; + pub const CONTEXT_3: u8 = 0xa3; +} + +/// A minimal reader for the subset of DER that X.509 certificates use. +/// +/// It decodes one tag length value triple at a time and hands back the content +/// as a borrowed slice. It deliberately does not understand the meaning of any +/// structure; the JA4X walk drives it, deciding which fields to descend into and +/// which to skip. Keeping the reader this small keeps it auditable, which +/// matters because certificate parsers are a classic source of memory safety +/// bugs and this one has no `unsafe` and cannot read out of bounds. +pub struct Der<'a> { + buf: &'a [u8], + pos: usize, +} + +impl<'a> Der<'a> { + #[must_use] + pub const fn new(buf: &'a [u8]) -> Self { + Self { buf, pos: 0 } + } + + #[must_use] + pub const fn is_empty(&self) -> bool { + self.pos >= self.buf.len() + } + + fn byte(&mut self) -> Result { + if self.pos >= self.buf.len() { + return Err(ParseError::Truncated { needed: 1, have: 0 }); + } + let b = self.buf[self.pos]; + self.pos += 1; + Ok(b) + } + + /// Reads one tag length value triple and returns the tag and the content + /// slice, advancing past the whole triple. + pub fn read_tlv(&mut self) -> Result<(u8, &'a [u8])> { + let tag = self.byte()?; + let len = self.read_length()?; + if self.buf.len() - self.pos < len { + return Err(ParseError::LengthOverrun { + field: "der", + declared: len, + available: self.buf.len() - self.pos, + }); + } + let content = &self.buf[self.pos..self.pos + len]; + self.pos += len; + Ok((tag, content)) + } + + fn read_length(&mut self) -> Result { + let first = self.byte()?; + if first & 0x80 == 0 { + return Ok(first as usize); + } + let count = (first & 0x7f) as usize; + if count == 0 || count > core::mem::size_of::() { + return Err(ParseError::Misaligned(count)); + } + let mut len = 0usize; + for _ in 0..count { + len = (len << 8) | self.byte()? as usize; + } + Ok(len) + } +} + +#[cfg(test)] +mod tests { + use super::{Der, tag}; + + #[test] + fn reads_short_form_sequence() { + let data = [0x30, 0x03, 0x06, 0x01, 0x55]; + let mut der = Der::new(&data); + let (t, content) = der.read_tlv().unwrap(); + assert_eq!(t, tag::SEQUENCE); + assert_eq!(content, &[0x06, 0x01, 0x55]); + assert!(der.is_empty()); + } + + #[test] + fn reads_long_form_length() { + let mut data = vec![0x04, 0x82, 0x01, 0x00]; + data.extend(std::iter::repeat_n(0xaa, 256)); + let mut der = Der::new(&data); + let (_t, content) = der.read_tlv().unwrap(); + assert_eq!(content.len(), 256); + } + + #[test] + fn rejects_length_overrun() { + let data = [0x06, 0x05, 0x55, 0x04]; + let mut der = Der::new(&data); + assert!(der.read_tlv().is_err()); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/error.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/error.rs new file mode 100644 index 00000000..df68f4fd --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/error.rs @@ -0,0 +1,57 @@ +// ©AngelaMos | 2026 +// error.rs + +use thiserror::Error; + +/// Errors produced while parsing TLS records, handshake messages, or QUIC +/// initial packets. +/// +/// Every variant is stack only. No variant carries a heap allocation, so the +/// malformed packet path never touches the allocator. This matters because a +/// fingerprinting engine spends most of its time rejecting traffic that is not +/// a clean handshake, and allocating on each rejection adds jitter at line rate. +#[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum ParseError { + #[error("buffer too short: needed {needed} bytes, had {have}")] + Truncated { needed: usize, have: usize }, + + #[error("not a TLS handshake record (content type {0:#04x})")] + NotHandshake(u8), + + #[error("unexpected handshake message type {0:#04x}")] + UnexpectedHandshake(u8), + + #[error("length field {field} declares {declared} bytes but {available} remain")] + LengthOverrun { + field: &'static str, + declared: usize, + available: usize, + }, + + #[error("vector length {0} is not a whole number of elements")] + Misaligned(usize), + + #[error("trailing {0} bytes after a complete message")] + Trailing(usize), + + #[error("handshake message spans more bytes than the reassembly cap allows")] + OversizedHandshake, + + #[error("malformed extension {ext_type:#06x}")] + BadExtension { ext_type: u16 }, + + #[error("not a QUIC long header initial packet")] + NotQuicInitial, + + #[error("unsupported QUIC version {0:#010x}")] + UnsupportedQuicVersion(u32), + + #[error("QUIC header protection or AEAD removal failed")] + QuicCryptoFailure, + + #[error("no CRYPTO frame carrying a ClientHello was present")] + NoCryptoClientHello, +} + +pub type Result = core::result::Result; diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/fingerprint.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/fingerprint.rs new file mode 100644 index 00000000..9ea4bbfe --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/fingerprint.rs @@ -0,0 +1,100 @@ +// ©AngelaMos | 2026 +// fingerprint.rs + +use core::fmt; + +use serde::Serialize; + +/// A JA3 or JA3S fingerprint: the MD5 digest of the pre hash string. +/// +/// JA3 is carried as the raw sixteen byte digest rather than a hex string so +/// that equality, hashing, and database keys operate on the compact binary form +/// and the hex rendering happens only at display boundaries. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize)] +#[serde(into = "String")] +pub struct Ja3([u8; 16]); + +impl Ja3 { + #[must_use] + pub const fn from_digest(digest: [u8; 16]) -> Self { + Self(digest) + } + + #[must_use] + pub const fn bytes(&self) -> &[u8; 16] { + &self.0 + } +} + +impl fmt::Display for Ja3 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for byte in self.0 { + write!(f, "{byte:02x}")?; + } + Ok(()) + } +} + +impl fmt::Debug for Ja3 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Ja3({self})") + } +} + +impl From for String { + fn from(value: Ja3) -> Self { + value.to_string() + } +} + +/// A fingerprint from the JA4 family, carried in both its hashed canonical form +/// and its raw pre hash form. +/// +/// The raw form is the unhashed list of cipher and extension values. It is kept +/// alongside the hash because it is the form an analyst reads when explaining +/// why two clients differ, and because clustering on the raw lists is more +/// informative than clustering on opaque digests. +#[derive(Clone, PartialEq, Eq, Hash, Serialize)] +pub struct Ja4Family { + pub hash: String, + pub raw: String, +} + +impl Ja4Family { + #[must_use] + pub fn new(hash: String, raw: String) -> Self { + Self { hash, raw } + } +} + +impl fmt::Display for Ja4Family { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.hash) + } +} + +impl fmt::Debug for Ja4Family { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Ja4Family(hash={}, raw={})", self.hash, self.raw) + } +} + +#[cfg(test)] +mod tests { + use super::{Ja3, Ja4Family}; + + #[test] + fn ja3_display_is_lowercase_hex() { + let fp = Ja3::from_digest([ + 0xad, 0xa7, 0x02, 0x06, 0xe4, 0x06, 0x42, 0xa3, 0xe4, 0x46, 0x1f, 0x35, 0x50, 0x32, + 0x41, 0xd5, + ]); + assert_eq!(fp.to_string(), "ada70206e40642a3e4461f35503241d5"); + } + + #[test] + fn ja4_family_displays_hash() { + let fp = Ja4Family::new("t13d1516h2_8daaf6152771_e5627efa2ab1".into(), "raw".into()); + assert_eq!(fp.to_string(), "t13d1516h2_8daaf6152771_e5627efa2ab1"); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/grease.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/grease.rs new file mode 100644 index 00000000..8d582220 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/grease.rs @@ -0,0 +1,52 @@ +// ©AngelaMos | 2026 +// grease.rs + +/// The sixteen GREASE values reserved by RFC 8701. +/// +/// GREASE (Generate Random Extensions And Sustain Extensibility) values are +/// inserted by clients into cipher lists, extension lists, supported groups, +/// supported versions, and signature algorithms. They are deliberate noise +/// whose only purpose is to keep middleboxes tolerant of unknown values. Both +/// JA3 and JA4 strip them before hashing so that the same client produces a +/// stable fingerprint regardless of which GREASE values it happened to pick. +/// +/// Every value has the form `0xZaZa` where the two bytes are equal and the low +/// nibble of each is `a`. +pub const GREASE_VALUES: [u16; 16] = [ + 0x0a0a, 0x1a1a, 0x2a2a, 0x3a3a, 0x4a4a, 0x5a5a, 0x6a6a, 0x7a7a, 0x8a8a, 0x9a9a, 0xaaaa, 0xbaba, + 0xcaca, 0xdada, 0xeaea, 0xfafa, +]; + +/// Returns true when `value` is one of the sixteen RFC 8701 GREASE values. +/// +/// The check exploits the structural regularity of the GREASE set rather than +/// scanning the table: both bytes must be equal and the low nibble of each must +/// be `0xa`. This is a single pair of comparisons rather than a sixteen way +/// branch, which keeps it cheap on the per packet path. +#[inline] +#[must_use] +pub const fn is_grease(value: u16) -> bool { + let high = (value >> 8) as u8; + let low = (value & 0x00ff) as u8; + high == low && (low & 0x0f) == 0x0a +} + +#[cfg(test)] +mod tests { + use super::{GREASE_VALUES, is_grease}; + + #[test] + fn table_matches_structural_check() { + for v in 0..=u16::MAX { + let in_table = GREASE_VALUES.contains(&v); + assert_eq!(in_table, is_grease(v), "mismatch for {v:#06x}"); + } + } + + #[test] + fn known_non_grease_values() { + for v in [0x0000_u16, 0x0010, 0x1301, 0x00ff, 0x5600, 0xc02f] { + assert!(!is_grease(v), "{v:#06x} should not be grease"); + } + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/hash.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/hash.rs new file mode 100644 index 00000000..849b87ae --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/hash.rs @@ -0,0 +1,26 @@ +// ©AngelaMos | 2026 +// hash.rs + +use sha2::{Digest, Sha256}; + +/// Returns the first twelve hex characters of the SHA256 digest of `bytes`. +/// +/// The whole JA4 family truncates SHA256 to twelve hex characters, which is six +/// bytes of digest. Twelve characters is enough to make accidental collisions +/// vanishingly unlikely across the fingerprint space while keeping the +/// fingerprint short enough to read and to paste into a search box. +#[must_use] +pub fn sha256_hex12(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + hex::encode(&digest[..6]) +} + +#[cfg(test)] +mod tests { + use super::sha256_hex12; + + #[test] + fn known_digest_prefix() { + assert_eq!(sha256_hex12(b""), "e3b0c44298fc"); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja3.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja3.rs new file mode 100644 index 00000000..0f530075 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja3.rs @@ -0,0 +1,125 @@ +// ©AngelaMos | 2026 +// ja3.rs + +use std::fmt::Write as _; + +use md5::{Digest, Md5}; +use smallvec::SmallVec; + +use crate::fingerprint::Ja3; +use crate::grease::is_grease; +use crate::parse::{ClientHello, ServerHello}; + +/// Computes the JA3 string and digest for a ClientHello. +/// +/// JA3 concatenates five fields, in order: the legacy version, the cipher +/// suites, the extension types, the supported groups, and the elliptic curve +/// point formats. Within a field the values are decimal and joined with +/// hyphens; the fields themselves are joined with commas. GREASE values are +/// removed from every list field so that the deliberate noise a client inserts +/// does not change its fingerprint. The MD5 of that string is the JA3 hash. +/// +/// JA3 is kept here despite being effectively dead for modern browser traffic, +/// because malware fingerprints in public feeds are still expressed as JA3 and +/// because watching JA3 collapse next to JA4 is the clearest way to understand +/// why JA4 exists. +#[must_use] +pub fn ja3_string(ch: &ClientHello) -> String { + let ext_types: SmallVec<[u16; 16]> = ch.extensions.iter().map(|e| e.ext_type).collect(); + let groups = ch.supported_groups(); + let formats = ch.ec_point_formats(); + + let mut s = String::new(); + let _ = write!(s, "{}", ch.legacy_version); + s.push(','); + append_u16_hyphenated(&mut s, &ch.cipher_suites); + s.push(','); + append_u16_hyphenated(&mut s, &ext_types); + s.push(','); + append_u16_hyphenated(&mut s, &groups); + s.push(','); + append_u8_hyphenated(&mut s, &formats); + s +} + +/// Computes the JA3 digest for a ClientHello. +#[must_use] +pub fn ja3(ch: &ClientHello) -> Ja3 { + digest(&ja3_string(ch)) +} + +/// Computes the JA3S string for a ServerHello. +/// +/// JA3S mirrors JA3 on the server side with three fields: the version, the +/// single chosen cipher suite, and the extension types. A server and the exact +/// ClientHello it answered together identify a deployment more tightly than +/// either side alone. +#[must_use] +pub fn ja3s_string(sh: &ServerHello) -> String { + let ext_types: SmallVec<[u16; 16]> = sh.extensions.iter().map(|e| e.ext_type).collect(); + let mut s = String::new(); + let _ = write!(s, "{},{},", sh.legacy_version, sh.cipher_suite); + append_u16_hyphenated(&mut s, &ext_types); + s +} + +/// Computes the JA3S digest for a ServerHello. +#[must_use] +pub fn ja3s(sh: &ServerHello) -> Ja3 { + digest(&ja3s_string(sh)) +} + +fn digest(pre_hash: &str) -> Ja3 { + let out = Md5::digest(pre_hash.as_bytes()); + let mut bytes = [0u8; 16]; + bytes.copy_from_slice(&out); + Ja3::from_digest(bytes) +} + +fn append_u16_hyphenated(out: &mut String, values: &[u16]) { + let mut first = true; + for &v in values { + if is_grease(v) { + continue; + } + if !first { + out.push('-'); + } + first = false; + let _ = write!(out, "{v}"); + } +} + +fn append_u8_hyphenated(out: &mut String, values: &[u8]) { + let mut first = true; + for &v in values { + if !first { + out.push('-'); + } + first = false; + let _ = write!(out, "{v}"); + } +} + +#[cfg(test)] +mod tests { + use super::digest; + + #[test] + fn salesforce_client_vector_one() { + let pre = "769,47-53-5-10-49161-49162-49171-49172-50-56-19-4,0-10-11,23-24-25,0"; + assert_eq!(digest(pre).to_string(), "ada70206e40642a3e4461f35503241d5"); + } + + #[test] + fn salesforce_client_vector_two_empty_fields() { + let pre = "769,4-5-10-9-100-98-3-6-19-18-99,,,"; + assert_eq!(digest(pre).to_string(), "de350869b8c85de67a350c8d186f11e6"); + } + + #[test] + fn server_vector_round_trips_through_md5() { + let pre = "769,47,65281-0-11-35-5-16"; + assert_eq!(digest(pre).to_string(), "836ce314215654b5b1f85f97c73e506f"); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4.rs new file mode 100644 index 00000000..f4629b4d --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4.rs @@ -0,0 +1,263 @@ +// ©AngelaMos | 2026 +// ja4.rs + +use std::fmt::Write as _; + +use smallvec::SmallVec; + +use crate::fingerprint::Ja4Family; +use crate::grease::is_grease; +use crate::hash::sha256_hex12; +use crate::parse::{ClientHello, ServerHello}; +use crate::registry::{extension, ja4_version_code}; + +/// The transport that carried the handshake. +/// +/// JA4 records the transport in its first character because the same TLS stack +/// produces a recognizably different ClientHello over QUIC than over TCP, and an +/// analyst wants to see that difference at a glance rather than infer it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Transport { + Tcp, + Quic, + Dtls, +} + +impl Transport { + const fn marker(self) -> char { + match self { + Transport::Tcp => 't', + Transport::Quic => 'q', + Transport::Dtls => 'd', + } + } +} + +const EMPTY_HASH: &str = "000000000000"; + +/// Computes the JA4 fingerprint of a ClientHello in both hashed and raw forms. +/// +/// The fingerprint has three underscore separated sections. The first is ten +/// human readable characters: transport, version, whether SNI was present, +/// cipher count, extension count, and two characters derived from the first +/// ALPN value. The second is a truncated SHA256 of the cipher list sorted by +/// value. The third is a truncated SHA256 of the extension list sorted by value, +/// with SNI and ALPN removed and the signature algorithms appended in their +/// original order. +/// +/// Sorting the cipher and extension lists before hashing is the whole reason +/// JA4 survived what killed JA3. When Chrome began shuffling its extension order +/// on every connection, JA3, which hashes extensions in wire order, produced a +/// fresh hash for every Chrome request. JA4 sorts first, so order shuffling +/// leaves the fingerprint unchanged. +#[must_use] +pub fn ja4(ch: &ClientHello, transport: Transport) -> Ja4Family { + let prefix = ja4_prefix(ch, transport); + + let (cipher_csv, _) = sorted_hex_csv(&ch.cipher_suites, &[]); + let cipher_hash = truncated_sha256(&cipher_csv); + + let ext_raw = ja4_extension_raw(ch); + let ext_hash = truncated_sha256(&ext_raw); + + let hash = format!("{prefix}_{cipher_hash}_{ext_hash}"); + let raw = format!("{prefix}_{cipher_csv}_{ext_raw}"); + Ja4Family::new(hash, raw) +} + +/// Computes the JA4S fingerprint of a ServerHello. +/// +/// JA4S mirrors JA4 on the server side, with three differences that follow from +/// what a server controls. The server picks exactly one cipher suite, so the +/// cipher section is that single value in hex rather than a hash of a list. The +/// extensions are hashed in the order the server sent them, not sorted, because +/// a server does not shuffle its own extensions. And there is no SNI field, +/// because the server is not the party naming a host. +#[must_use] +pub fn ja4s(sh: &ServerHello, transport: Transport) -> Ja4Family { + let version = ja4_version_code(sh.selected_version()); + let alpn = alpn_chars(sh.alpn_protocol()); + let ext_count = sh.extensions.len().min(99); + + let mut prefix = String::with_capacity(7); + prefix.push(transport.marker()); + prefix.push_str(version); + let _ = write!(prefix, "{ext_count:02}"); + prefix.push_str(&alpn); + + let cipher_hex = format!("{:04x}", sh.cipher_suite); + + let ext_csv = wire_order_hex_csv( + &sh.extensions + .iter() + .map(|e| e.ext_type) + .collect::>(), + ); + let ext_hash = truncated_sha256(&ext_csv); + + let hash = format!("{prefix}_{cipher_hex}_{ext_hash}"); + let raw = format!("{prefix}_{cipher_hex}_{ext_csv}"); + Ja4Family::new(hash, raw) +} + +fn wire_order_hex_csv(values: &[u16]) -> String { + let hexed: SmallVec<[String; 16]> = values.iter().map(|v| format!("{v:04x}")).collect(); + hexed.join(",") +} + +fn ja4_prefix(ch: &ClientHello, transport: Transport) -> String { + let version = select_version(ch); + let sni = if ch.has_extension(extension::SERVER_NAME) { + 'd' + } else { + 'i' + }; + let cipher_count = capped_count(ch.cipher_suites.iter().copied()); + let ext_count = capped_count(ch.extensions.iter().map(|e| e.ext_type)); + let alpn = ja4_alpn(ch); + + let mut prefix = String::with_capacity(10); + prefix.push(transport.marker()); + prefix.push_str(ja4_version_code(version)); + prefix.push(sni); + let _ = write!(prefix, "{cipher_count:02}{ext_count:02}"); + prefix.push_str(&alpn); + prefix +} + +/// Selects the JA4 version word: the highest non GREASE value from the supported +/// versions extension when present, otherwise the legacy record version. +fn select_version(ch: &ClientHello) -> u16 { + ch.supported_versions() + .iter() + .copied() + .filter(|v| !is_grease(*v)) + .max() + .unwrap_or(ch.legacy_version) +} + +fn capped_count(values: impl Iterator) -> usize { + values.filter(|v| !is_grease(*v)).count().min(99) +} + +/// Derives the two ALPN characters. +/// +/// The implementation follows the published JA4 specification rather than the +/// FoxIO Python reference. The two diverge for ALPN values whose first or last +/// byte is not an ASCII alphanumeric: the specification says to print the first +/// and last characters of the hex encoding of the value, while the Python +/// reference emits a fixed fallback. The specification is the more informative +/// and more portable choice, so it is what this code does. +fn ja4_alpn(ch: &ClientHello) -> String { + alpn_chars(ch.alpn_protocols().first().copied()) +} + +fn alpn_chars(first: Option<&[u8]>) -> String { + let Some(first) = first else { + return "00".to_string(); + }; + if first.is_empty() { + return "00".to_string(); + } + + let first_byte = first[0]; + let last_byte = first[first.len() - 1]; + + if is_ascii_alphanumeric(first_byte) && is_ascii_alphanumeric(last_byte) { + let mut out = String::with_capacity(2); + out.push(first_byte as char); + out.push(last_byte as char); + out + } else { + let encoded = hex::encode(first); + let mut chars = encoded.chars(); + let first_char = chars.next().unwrap_or('0'); + let last_char = encoded.chars().last().unwrap_or('0'); + let mut out = String::with_capacity(2); + out.push(first_char); + out.push(last_char); + out + } +} + +const fn is_ascii_alphanumeric(byte: u8) -> bool { + byte.is_ascii_digit() || byte.is_ascii_uppercase() || byte.is_ascii_lowercase() +} + +/// Builds the raw, pre hash extension string for section three. +/// +/// The extension types are sorted by hex value after removing GREASE, SNI, and +/// ALPN. If a signature algorithms extension is present, its values are appended +/// after an underscore, in their original order. If it is absent, the string +/// ends without a trailing underscore, which is the behavior the specification +/// requires and which changes the resulting hash. +fn ja4_extension_raw(ch: &ClientHello) -> String { + let excluded = [extension::SERVER_NAME, extension::ALPN]; + let ext_types: SmallVec<[u16; 16]> = ch.extensions.iter().map(|e| e.ext_type).collect(); + let (ext_csv, _) = sorted_hex_csv(&ext_types, &excluded); + + let sig_algs = ch.signature_algorithms(); + if ch.has_extension(extension::SIGNATURE_ALGORITHMS) { + let sig_csv = unsorted_hex_csv(&sig_algs); + format!("{ext_csv}_{sig_csv}") + } else { + ext_csv + } +} + +fn sorted_hex_csv(values: &[u16], excluded: &[u16]) -> (String, usize) { + let mut hexed: SmallVec<[String; 32]> = values + .iter() + .copied() + .filter(|v| !is_grease(*v) && !excluded.contains(v)) + .map(|v| format!("{v:04x}")) + .collect(); + hexed.sort_unstable(); + (hexed.join(","), hexed.len()) +} + +fn unsorted_hex_csv(values: &[u16]) -> String { + let hexed: SmallVec<[String; 16]> = values + .iter() + .copied() + .filter(|v| !is_grease(*v)) + .map(|v| format!("{v:04x}")) + .collect(); + hexed.join(",") +} + +fn truncated_sha256(input: &str) -> String { + if input.is_empty() { + return EMPTY_HASH.to_string(); + } + sha256_hex12(input.as_bytes()) +} + +#[cfg(test)] +mod tests { + use super::{EMPTY_HASH, truncated_sha256}; + + #[test] + fn empty_input_is_the_zero_hash() { + assert_eq!(truncated_sha256(""), EMPTY_HASH); + } + + #[test] + fn truncation_is_twelve_hex_chars() { + let h = truncated_sha256("1301,1302,1303"); + assert_eq!(h.len(), 12); + assert!(h.chars().all(|c| c.is_ascii_hexdigit())); + } + + #[test] + fn foxio_cipher_section_vector() { + let ciphers = "002f,0035,009c,009d,1301,1302,1303,c013,c014,c02b,c02c,c02f,c030,cca8,cca9"; + assert_eq!(truncated_sha256(ciphers), "8daaf6152771"); + } + + #[test] + fn foxio_extension_section_vector() { + let exts = "0005,000a,000b,000d,0012,0015,0017,001b,0023,002b,002d,0033,4469,ff01_0403,0804,0401,0503,0805,0501,0806,0601"; + assert_eq!(truncated_sha256(exts), "e5627efa2ab1"); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4h.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4h.rs new file mode 100644 index 00000000..be52079b --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4h.rs @@ -0,0 +1,195 @@ +// ©AngelaMos | 2026 +// ja4h.rs + +use crate::fingerprint::Ja4Family; +use crate::hash::sha256_hex12; + +/// A parsed HTTP request, holding what JA4H reads. +/// +/// JA4H fingerprints an HTTP client from one request: its method, version, +/// whether it carries cookies and a referer, the names of its other headers in +/// the order they were sent, its accept language, and its cookie names and +/// values. A request that omits an accept language and sends no cookies is far +/// more likely to be a script than a person, and JA4H makes that visible in the +/// first few characters. +/// +/// This applies to cleartext HTTP only. Over HTTPS the request is encrypted and +/// invisible to a passive observer, and HTTP/2 carries its headers in HPACK, +/// which a passive tool cannot decode without following the whole connection. +#[derive(Debug, Clone)] +pub struct HttpRequest { + pub method: String, + pub version: String, + pub headers: Vec<(String, String)>, +} + +/// Parses an HTTP/1.x request from the start of a reassembled byte stream. +/// +/// Returns `None` when the bytes are not a well formed request line followed by +/// headers. Header values keep their original bytes; header names keep their +/// original case because JA4H hashes them as sent. +#[must_use] +pub fn parse_http_request(bytes: &[u8]) -> Option { + let text = std::str::from_utf8(bytes).ok()?; + let mut lines = text.split("\r\n"); + + let request_line = lines.next()?; + let mut parts = request_line.split(' '); + let method = parts.next()?.to_string(); + let _target = parts.next()?; + let http_token = parts.next()?; + if !http_token.starts_with("HTTP/") { + return None; + } + let version = http_token.trim_start_matches("HTTP/").replace('.', ""); + + let mut headers = Vec::new(); + for line in lines { + if line.is_empty() { + break; + } + let (name, value) = line.split_once(':')?; + headers.push((name.to_string(), value.trim_start().to_string())); + } + + Some(HttpRequest { + method, + version, + headers, + }) +} + +/// Computes the JA4H fingerprint for a parsed HTTP request. +#[must_use] +pub fn ja4h(req: &HttpRequest) -> Ja4Family { + let method = method_code(&req.method); + let version = version_code(&req.version); + let cookie_flag = if has_header(req, "cookie") { 'c' } else { 'n' }; + let referer_flag = if has_named_header(req, "referer") { + 'r' + } else { + 'n' + }; + + let counted: Vec<&str> = req + .headers + .iter() + .map(|(name, _)| name.as_str()) + .filter(|name| is_counted_header(name)) + .collect(); + let header_len = counted.len().min(99); + let lang = accept_language(req); + + let prefix = format!("{method}{version}{cookie_flag}{referer_flag}{header_len:02}{lang}"); + let header_hash = sha12(&counted.join(",")); + + let cookies = cookie_pairs(req); + let (cookie_hash, value_hash, raw_cookie_tail) = if let Some(mut pairs) = cookies { + pairs.sort_by(|a, b| a.0.cmp(&b.0)); + let names: Vec<&str> = pairs.iter().map(|p| p.0.as_str()).collect(); + let entries: Vec<&str> = pairs.iter().map(|p| p.1.as_str()).collect(); + let tail = format!("{}_{}", names.join(","), entries.join(",")); + (sha12(&names.join(",")), sha12(&entries.join(",")), tail) + } else { + (ZERO_HASH.to_string(), ZERO_HASH.to_string(), String::new()) + }; + + let hash = format!("{prefix}_{header_hash}_{cookie_hash}_{value_hash}"); + let raw = format!("{prefix}_{}_{raw_cookie_tail}", counted.join(",")); + Ja4Family::new(hash, raw) +} + +const ZERO_HASH: &str = "000000000000"; + +fn method_code(method: &str) -> String { + method.to_lowercase().chars().take(2).collect() +} + +fn version_code(version: &str) -> String { + match version { + "2" | "20" => "20".to_string(), + "3" | "30" => "30".to_string(), + "10" => "10".to_string(), + _ => "11".to_string(), + } +} + +fn has_header(req: &HttpRequest, prefix_lower: &str) -> bool { + req.headers + .iter() + .any(|(name, _)| name.to_lowercase().starts_with(prefix_lower)) +} + +fn has_named_header(req: &HttpRequest, name_lower: &str) -> bool { + req.headers + .iter() + .any(|(name, _)| name.to_lowercase() == name_lower) +} + +fn is_counted_header(name: &str) -> bool { + let lower = name.to_lowercase(); + !name.starts_with(':') && !lower.starts_with("cookie") && lower != "referer" && !name.is_empty() +} + +fn accept_language(req: &HttpRequest) -> String { + let Some((_, value)) = req + .headers + .iter() + .find(|(name, _)| name.to_lowercase() == "accept-language") + else { + return "0000".to_string(); + }; + let normalized = value.replace('-', "").replace(';', ",").to_lowercase(); + let first = normalized.split(',').next().unwrap_or(""); + let mut code: String = first.chars().take(4).collect(); + while code.len() < 4 { + code.push('0'); + } + code +} + +fn cookie_pairs(req: &HttpRequest) -> Option> { + let (_, value) = req + .headers + .iter() + .find(|(name, _)| name.to_lowercase() == "cookie")?; + let pairs = value + .split(';') + .map(str::trim) + .filter(|p| !p.is_empty()) + .map(|p| { + let name = p.split('=').next().unwrap_or(p).trim().to_string(); + (name, p.to_string()) + }) + .collect(); + Some(pairs) +} + +fn sha12(joined: &str) -> String { + sha256_hex12(joined.as_bytes()) +} + +#[cfg(test)] +mod tests { + use super::{parse_http_request, version_code}; + + #[test] + fn parses_request_line_and_headers() { + let raw = b"GET /index.html HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n"; + let req = parse_http_request(raw).unwrap(); + assert_eq!(req.method, "GET"); + assert_eq!(req.version, "11"); + assert_eq!(req.headers.len(), 2); + assert_eq!( + req.headers[0], + ("Host".to_string(), "example.com".to_string()) + ); + } + + #[test] + fn version_codes() { + assert_eq!(version_code("11"), "11"); + assert_eq!(version_code("10"), "10"); + assert_eq!(version_code("2"), "20"); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4t.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4t.rs new file mode 100644 index 00000000..7cdc7609 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4t.rs @@ -0,0 +1,81 @@ +// ©AngelaMos | 2026 +// ja4t.rs + +use std::fmt::Write as _; + +use smallvec::SmallVec; + +/// The TCP layer inputs to a JA4T fingerprint, read from a SYN or SYN ACK. +/// +/// JA4T fingerprints the TCP stack rather than the TLS stack. It is computed +/// from fields that an operating system sets the same way on every connection +/// but that differ between operating systems: the advertised window size, the +/// kinds and order of TCP options, the maximum segment size, and the window +/// scale factor. Pairing it with JA4 exposes a class of evasion that TLS only +/// fingerprinting misses, where a tool wears a browser's TLS clothing while its +/// host operating system speaks with a different TCP accent. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TcpFingerprintInput { + pub window_size: u16, + pub option_kinds: SmallVec<[u8; 8]>, + pub mss: u16, + pub window_scale: u8, +} + +/// Computes the JA4T or JA4TS string. +/// +/// The format is the window size, then the option kinds joined with hyphens, +/// then the maximum segment size, then the window scale, with the four parts +/// separated by underscores. A missing MSS or window scale option is reported as +/// zero. The same function serves both the client SYN, which yields JA4T, and +/// the server SYN ACK, which yields JA4TS. +#[must_use] +pub fn ja4t(input: &TcpFingerprintInput) -> String { + let mut options = String::new(); + let mut first = true; + for kind in &input.option_kinds { + if !first { + options.push('-'); + } + first = false; + let _ = write!(options, "{kind}"); + } + + format!( + "{}_{}_{}_{}", + input.window_size, options, input.mss, input.window_scale + ) +} + +#[cfg(test)] +mod tests { + use super::{TcpFingerprintInput, ja4t}; + use smallvec::SmallVec; + + fn input(window: u16, kinds: &[u8], mss: u16, scale: u8) -> TcpFingerprintInput { + TcpFingerprintInput { + window_size: window, + option_kinds: SmallVec::from_slice(kinds), + mss, + window_scale: scale, + } + } + + #[test] + fn foxio_linux_default_vector() { + let i = input(64240, &[2, 1, 3, 1, 1, 4], 1460, 8); + assert_eq!(ja4t(&i), "64240_2-1-3-1-1-4_1460_8"); + } + + #[test] + fn foxio_windowed_vector() { + let i = input(65535, &[2, 1, 3, 1, 1, 8, 4, 0, 0], 1346, 6); + assert_eq!(ja4t(&i), "65535_2-1-3-1-1-8-4-0-0_1346_6"); + } + + #[test] + fn missing_mss_and_scale_report_zero() { + let i = input(8192, &[2, 4], 0, 0); + assert_eq!(ja4t(&i), "8192_2-4_0_0"); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4x.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4x.rs new file mode 100644 index 00000000..df2af350 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4x.rs @@ -0,0 +1,175 @@ +// ©AngelaMos | 2026 +// ja4x.rs + +use smallvec::SmallVec; + +use crate::der::{Der, tag}; +use crate::error::{ParseError, Result}; +use crate::hash::sha256_hex12; + +/// Computes the JA4X fingerprint of one DER encoded X.509 certificate. +/// +/// JA4X does not fingerprint the contents of a certificate. It fingerprints how +/// the certificate was built: which object identifiers appear in the issuer +/// name, which appear in the subject name, and which appear among the +/// extensions, each in the order they were written. Two certificates minted by +/// the same software with the same template share a JA4X even when every name +/// and serial differs, which is what makes it useful for clustering certificates +/// from one toolchain or one malware family. +/// +/// Passively this only works on TLS 1.2 and earlier, where the certificate +/// travels in the clear. TLS 1.3 encrypts the certificate message, so a passive +/// observer never sees it. +pub fn ja4x(cert_der: &[u8]) -> Result { + let (issuer_oids, subject_oids, ext_oids) = certificate_oids(cert_der)?; + + let issuer_hash = sha256_hex12(issuer_oids.join(",").as_bytes()); + let subject_hash = sha256_hex12(subject_oids.join(",").as_bytes()); + let ext_hash = sha256_hex12(ext_oids.join(",").as_bytes()); + + Ok(format!("{issuer_hash}_{subject_hash}_{ext_hash}")) +} + +type OidList = SmallVec<[String; 8]>; + +fn certificate_oids(cert_der: &[u8]) -> Result<(OidList, OidList, OidList)> { + let mut outer = Der::new(cert_der); + let (_, certificate) = outer.read_tlv()?; + let mut cert = Der::new(certificate); + let (_, tbs) = cert.read_tlv()?; + + let mut fields: SmallVec<[(u8, &[u8]); 10]> = SmallVec::new(); + let mut walker = Der::new(tbs); + while !walker.is_empty() { + fields.push(walker.read_tlv()?); + } + + let mut idx = 0; + if fields.first().is_some_and(|(t, _)| *t == 0xa0) { + idx += 1; + } + idx += 2; + let issuer = field_content(&fields, idx)?; + idx += 1; + idx += 1; + let subject = field_content(&fields, idx)?; + idx += 1; + idx += 1; + + let extensions = fields[idx..] + .iter() + .find(|(t, _)| *t == tag::CONTEXT_3) + .map(|(_, c)| *c); + + let issuer_oids = name_oids(issuer)?; + let subject_oids = name_oids(subject)?; + let ext_oids = match extensions { + Some(content) => extension_oids(content)?, + None => OidList::new(), + }; + + Ok((issuer_oids, subject_oids, ext_oids)) +} + +fn field_content<'a>(fields: &[(u8, &'a [u8])], idx: usize) -> Result<&'a [u8]> { + fields + .get(idx) + .map(|(_, c)| *c) + .ok_or(ParseError::Truncated { needed: 1, have: 0 }) +} + +/// Collects the attribute type object identifiers from a Name, in order. +/// +/// A Name is a sequence of relative distinguished names, each a set of attribute +/// type and value pairs. The fingerprint reads only the attribute type, the +/// leading object identifier of each pair, and renders it as the hex of its DER +/// content bytes, which is exactly the form JA4X hashes. +fn name_oids(name: &[u8]) -> Result { + let mut oids = OidList::new(); + let mut rdns = Der::new(name); + while !rdns.is_empty() { + let (_, rdn) = rdns.read_tlv()?; + let mut set = Der::new(rdn); + while !set.is_empty() { + let (_, atv) = set.read_tlv()?; + let mut pair = Der::new(atv); + let (oid_tag, oid) = pair.read_tlv()?; + if oid_tag == tag::OBJECT_IDENTIFIER { + oids.push(hex::encode(oid)); + } + } + } + Ok(oids) +} + +/// Collects the extension object identifiers, in order. +fn extension_oids(context: &[u8]) -> Result { + let mut oids = OidList::new(); + let mut wrapper = Der::new(context); + let (_, sequence) = wrapper.read_tlv()?; + let mut exts = Der::new(sequence); + while !exts.is_empty() { + let (_, ext) = exts.read_tlv()?; + let mut fields = Der::new(ext); + let (oid_tag, oid) = fields.read_tlv()?; + if oid_tag == tag::OBJECT_IDENTIFIER { + oids.push(hex::encode(oid)); + } + } + Ok(oids) +} + +#[cfg(test)] +mod tests { + use super::{extension_oids, name_oids}; + + fn oid(content: &[u8]) -> Vec { + let mut v = vec![0x06, u8::try_from(content.len()).unwrap()]; + v.extend_from_slice(content); + v + } + + fn tlv(tag: u8, content: &[u8]) -> Vec { + let mut v = vec![tag, u8::try_from(content.len()).unwrap()]; + v.extend_from_slice(content); + v + } + + #[test] + fn extracts_issuer_oids_in_order() { + let mut name = Vec::new(); + for content in [ + &[0x55u8, 0x04, 0x06][..], + &[0x55, 0x04, 0x0a], + &[0x55, 0x04, 0x0b], + &[0x55, 0x04, 0x03], + ] { + let atv = tlv(0x30, &{ + let mut a = oid(content); + a.extend(tlv(0x13, b"x")); + a + }); + name.extend(tlv(0x31, &atv)); + } + + let oids = name_oids(&name).unwrap(); + assert_eq!(oids.as_slice(), &["550406", "55040a", "55040b", "550403"]); + } + + #[test] + fn extracts_extension_oids() { + let mut sequence = Vec::new(); + for content in [&[0x55u8, 0x1d, 0x23][..], &[0x55, 0x1d, 0x0e]] { + let ext = tlv(0x30, &{ + let mut e = oid(content); + e.extend(tlv(0x04, b"v")); + e + }); + sequence.extend(ext); + } + let context = tlv(0x30, &sequence); + + let oids = extension_oids(&context).unwrap(); + assert_eq!(oids.as_slice(), &["551d23", "551d0e"]); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/lib.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/lib.rs new file mode 100644 index 00000000..50fb0aed --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/lib.rs @@ -0,0 +1,37 @@ +// ©AngelaMos | 2026 +// lib.rs + +//! TLS handshake parsing and JA3/JA4 family fingerprint computation. +//! +//! This crate is the engine. It parses TLS records and handshake messages, the +//! TLS carried inside QUIC initial packets, and computes the JA3, JA3S, JA4, +//! JA4S, JA4H, JA4X, and JA4T fingerprints. It depends on nothing that touches +//! a network interface, a database, or an async runtime, so it can be embedded, +//! fuzzed, and unit tested in isolation. + +pub mod der; +pub mod error; +pub mod fingerprint; +pub mod grease; +pub mod hash; +pub mod ja3; +pub mod ja4; +pub mod ja4h; +pub mod ja4t; +pub mod ja4x; +pub mod parse; +pub mod pipeline; +pub mod registry; + +pub use error::{ParseError, Result}; +pub use fingerprint::{Ja3, Ja4Family}; +pub use grease::{GREASE_VALUES, is_grease}; +pub use ja3::{ja3, ja3_string, ja3s, ja3s_string}; +pub use ja4::{Transport, ja4, ja4s}; +pub use ja4h::{HttpRequest, ja4h, parse_http_request}; +pub use ja4t::{TcpFingerprintInput, ja4t}; +pub use ja4x::ja4x; +pub use parse::{ClientHello, Extension, ServerHello}; +pub use pipeline::event::{FingerprintEvent, StreamEvent}; +pub use pipeline::source::{PacketSource, PcapFileSource, RawFrame, SourceError}; +pub use pipeline::{Counters, Pipeline, PipelineConfig}; diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/cert.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/cert.rs new file mode 100644 index 00000000..5545777c --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/cert.rs @@ -0,0 +1,70 @@ +// ©AngelaMos | 2026 +// cert.rs + +use smallvec::SmallVec; + +use crate::error::Result; +use crate::parse::reader::Reader; + +/// The DER certificates carried by one TLS Certificate handshake message. +pub type CertificateList<'pkt> = SmallVec<[&'pkt [u8]; 4]>; + +/// Extracts the DER encoded certificates from a Certificate message body. +/// +/// The body is a three byte total length followed by entries that are each a +/// three byte length and the raw DER bytes. This is the TLS 1.2 framing; it is +/// the only framing a passive observer ever parses, because TLS 1.3 moved the +/// Certificate message inside the encrypted part of the handshake. The +/// certificates come back in wire order, leaf first, which is the order JA4X +/// reports them in. +pub fn certificate_der_list(body: &[u8]) -> Result> { + let mut r = Reader::new(body); + let mut list = r.sub_u24_vec()?; + let mut certs = CertificateList::new(); + while !list.is_empty() { + certs.push(list.take_u24_vec()?); + } + Ok(certs) +} + +#[cfg(test)] +mod tests { + use super::certificate_der_list; + use crate::error::ParseError; + + fn message(certs: &[&[u8]]) -> Vec { + let total: usize = certs.iter().map(|c| c.len() + 3).sum(); + let mut v = Vec::new(); + v.extend_from_slice(&u32::try_from(total).unwrap().to_be_bytes()[1..]); + for cert in certs { + v.extend_from_slice(&u32::try_from(cert.len()).unwrap().to_be_bytes()[1..]); + v.extend_from_slice(cert); + } + v + } + + #[test] + fn splits_a_two_certificate_chain() { + let body = message(&[&[0x30, 0x01, 0xaa], &[0x30, 0x02, 0xbb, 0xcc]]); + let certs = certificate_der_list(&body).unwrap(); + assert_eq!(certs.len(), 2); + assert_eq!(certs[0], &[0x30, 0x01, 0xaa]); + assert_eq!(certs[1], &[0x30, 0x02, 0xbb, 0xcc]); + } + + #[test] + fn truncated_entry_is_an_error_not_a_panic() { + let mut body = message(&[&[0x30, 0x01, 0xaa]]); + body.truncate(body.len() - 1); + assert!(matches!( + certificate_der_list(&body), + Err(ParseError::Truncated { .. }) + )); + } + + #[test] + fn empty_chain_is_empty() { + let certs = certificate_der_list(&[0, 0, 0]).unwrap(); + assert!(certs.is_empty()); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/hello.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/hello.rs new file mode 100644 index 00000000..39b32d7a --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/hello.rs @@ -0,0 +1,246 @@ +// ©AngelaMos | 2026 +// hello.rs + +use smallvec::SmallVec; + +use crate::error::Result; +use crate::parse::reader::Reader; +use crate::registry::extension; + +/// A single TLS extension as it appeared on the wire. +/// +/// The extension body is borrowed from the packet, not copied. Specific +/// extensions are decoded on demand through the accessor methods on +/// [`ClientHello`] so that the common parse path does no work for extensions a +/// given fingerprint does not read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Extension<'pkt> { + pub ext_type: u16, + pub data: &'pkt [u8], +} + +/// A parsed ClientHello, holding exactly what the fingerprint algorithms read. +/// +/// Cipher suites and extensions are stored in wire order. Order is preserved +/// because JA3 hashes extensions in their original order, and because the +/// difference between wire order and sorted order is itself a signal: a client +/// that permutes its extension order on every connection is doing something a +/// fixed order client is not. +#[derive(Debug, Clone)] +pub struct ClientHello<'pkt> { + pub legacy_version: u16, + pub cipher_suites: SmallVec<[u16; 32]>, + pub extensions: SmallVec<[Extension<'pkt>; 16]>, + pub is_sslv2: bool, +} + +/// A parsed ServerHello. +/// +/// The server selects exactly one cipher suite, so `cipher_suite` is a single +/// value rather than a list. Extensions are again kept in wire order. +#[derive(Debug, Clone)] +pub struct ServerHello<'pkt> { + pub legacy_version: u16, + pub cipher_suite: u16, + pub extensions: SmallVec<[Extension<'pkt>; 16]>, +} + +fn parse_extensions<'pkt>(r: &mut Reader<'pkt>) -> Result; 16]>> { + let mut exts = SmallVec::new(); + if r.is_empty() { + return Ok(exts); + } + let mut block = r.sub_u16_vec()?; + while !block.is_empty() { + let ext_type = block.u16()?; + let data = block.take_u16_vec()?; + exts.push(Extension { ext_type, data }); + } + Ok(exts) +} + +/// Parses the body of a ClientHello handshake message. +/// +/// The input is the message body, meaning the four byte handshake header has +/// already been stripped. The session id, which fingerprinting ignores, is +/// skipped, as are the compression methods; getting those two skips right is a +/// classic source of bugs because a parser that forgets them reads the wrong +/// bytes as cipher suites. +pub fn parse_client_hello(body: &[u8]) -> Result> { + let mut r = Reader::new(body); + let legacy_version = r.u16()?; + let _random = r.take(32)?; + let _session_id = r.take_u8_vec()?; + + let mut cipher_reader = r.sub_u16_vec()?; + let mut cipher_suites = SmallVec::new(); + while !cipher_reader.is_empty() { + cipher_suites.push(cipher_reader.u16()?); + } + + let _compression = r.take_u8_vec()?; + let extensions = parse_extensions(&mut r)?; + + Ok(ClientHello { + legacy_version, + cipher_suites, + extensions, + is_sslv2: false, + }) +} + +/// Parses the body of a ServerHello handshake message. +pub fn parse_server_hello(body: &[u8]) -> Result> { + let mut r = Reader::new(body); + let legacy_version = r.u16()?; + let _random = r.take(32)?; + let _session_id = r.take_u8_vec()?; + let cipher_suite = r.u16()?; + let _compression = r.u8()?; + let extensions = parse_extensions(&mut r)?; + + Ok(ServerHello { + legacy_version, + cipher_suite, + extensions, + }) +} + +impl<'pkt> ServerHello<'pkt> { + fn extension(&self, ext_type: u16) -> Option<&Extension<'pkt>> { + self.extensions.iter().find(|e| e.ext_type == ext_type) + } + + /// Returns the version the server selected. + /// + /// In TLS 1.3 the negotiated version lives in the supported versions + /// extension as a single value rather than in the legacy version word, which + /// the server pins to TLS 1.2 for compatibility. JA4S reads the real version + /// from the extension when it is present. + #[must_use] + pub fn selected_version(&self) -> u16 { + self.extension(extension::SUPPORTED_VERSIONS) + .and_then(|ext| { + let mut r = Reader::new(ext.data); + r.u16().ok() + }) + .unwrap_or(self.legacy_version) + } + + /// Returns the ALPN protocol the server chose, if any. + #[must_use] + pub fn alpn_protocol(&self) -> Option<&'pkt [u8]> { + let ext = self.extension(extension::ALPN)?; + let mut r = Reader::new(ext.data); + let mut list = r.sub_u16_vec().ok()?; + list.take_u8_vec().ok().filter(|p| !p.is_empty()) + } +} + +impl<'pkt> ClientHello<'pkt> { + fn extension(&self, ext_type: u16) -> Option<&Extension<'pkt>> { + self.extensions.iter().find(|e| e.ext_type == ext_type) + } + + #[must_use] + pub fn has_extension(&self, ext_type: u16) -> bool { + self.extension(ext_type).is_some() + } + + /// Returns the server name from the SNI extension, if a host name entry is + /// present. + #[must_use] + pub fn server_name(&self) -> Option<&'pkt str> { + let ext = self.extension(extension::SERVER_NAME)?; + let mut list = Reader::new(ext.data).sub_u16_vec().ok()?; + while !list.is_empty() { + let name_type = list.u8().ok()?; + let name = list.take_u16_vec().ok()?; + if name_type == 0 { + return core::str::from_utf8(name).ok(); + } + } + None + } + + /// Returns the supported groups, the field JA3 calls elliptic curves. + #[must_use] + pub fn supported_groups(&self) -> SmallVec<[u16; 16]> { + self.u16_list(extension::SUPPORTED_GROUPS) + } + + /// Returns the elliptic curve point formats. + #[must_use] + pub fn ec_point_formats(&self) -> SmallVec<[u8; 4]> { + let mut out = SmallVec::new(); + let Some(ext) = self.extension(extension::EC_POINT_FORMATS) else { + return out; + }; + let mut r = Reader::new(ext.data); + let Ok(list) = r.take_u8_vec() else { + return out; + }; + out.extend_from_slice(list); + out + } + + /// Returns the protocol versions the client offers in the supported versions + /// extension. JA4 selects its version field from the highest non GREASE + /// value here when the extension is present. + #[must_use] + pub fn supported_versions(&self) -> SmallVec<[u16; 8]> { + let mut out = SmallVec::new(); + let Some(ext) = self.extension(extension::SUPPORTED_VERSIONS) else { + return out; + }; + let mut r = Reader::new(ext.data); + let Ok(list) = r.take_u8_vec() else { + return out; + }; + let mut inner = Reader::new(list); + while let Ok(v) = inner.u16() { + out.push(v); + } + out + } + + /// Returns the signature algorithms in their original order. JA4 appends + /// these, unsorted, to the extension hash input. + #[must_use] + pub fn signature_algorithms(&self) -> SmallVec<[u16; 16]> { + self.u16_list(extension::SIGNATURE_ALGORITHMS) + } + + /// Returns the ALPN protocol identifiers in order. JA4 uses the first one. + #[must_use] + pub fn alpn_protocols(&self) -> SmallVec<[&'pkt [u8]; 4]> { + let mut out = SmallVec::new(); + let Some(ext) = self.extension(extension::ALPN) else { + return out; + }; + let mut r = Reader::new(ext.data); + let Ok(mut list) = r.sub_u16_vec() else { + return out; + }; + while let Ok(proto) = list.take_u8_vec() { + out.push(proto); + } + out + } + + fn u16_list(&self, ext_type: u16) -> SmallVec<[u16; 16]> { + let mut out = SmallVec::new(); + let Some(ext) = self.extension(ext_type) else { + return out; + }; + let mut r = Reader::new(ext.data); + let Ok(list) = r.take_u16_vec() else { + return out; + }; + let mut inner = Reader::new(list); + while let Ok(v) = inner.u16() { + out.push(v); + } + out + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/mod.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/mod.rs new file mode 100644 index 00000000..cd44327d --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/mod.rs @@ -0,0 +1,23 @@ +// ©AngelaMos | 2026 +// mod.rs + +//! Hand written, bounds checked TLS parsing. +//! +//! The parser is deliberately not built on a parser combinator framework. It is +//! the security critical core of the tool, and the byte by byte reader here maps +//! directly onto the wire format described in the TLS RFCs, which makes it easy +//! to audit against the specification. Reassembly of fragmented handshakes is +//! handled before parsing, so every parse function sees a complete message and +//! never has to model partial input. + +pub mod cert; +pub mod hello; +pub mod reader; +pub mod record; + +pub use cert::{CertificateList, certificate_der_list}; +pub use hello::{ClientHello, Extension, ServerHello, parse_client_hello, parse_server_hello}; +pub use reader::Reader; +pub use record::{ + first_handshake_message, handshake_bytes, is_sslv2_client_hello, parse_sslv2_client_hello, +}; diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/reader.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/reader.rs new file mode 100644 index 00000000..b4a8ccfc --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/reader.rs @@ -0,0 +1,173 @@ +// ©AngelaMos | 2026 +// reader.rs + +use crate::error::{ParseError, Result}; + +/// A forward only cursor over a borrowed byte slice with bounds checked reads. +/// +/// Every read advances the cursor and returns an error rather than panicking +/// when the buffer is too short. This is the foundation the whole parser stands +/// on: because the cursor can never read past the end of the slice, the parser +/// has no `unsafe`, cannot index out of bounds, and treats a truncated or +/// hostile packet as an ordinary error instead of a crash. The slices it hands +/// back borrow from the original packet buffer, so parsing copies nothing. +pub struct Reader<'pkt> { + buf: &'pkt [u8], + pos: usize, +} + +impl<'pkt> Reader<'pkt> { + #[must_use] + pub const fn new(buf: &'pkt [u8]) -> Self { + Self { buf, pos: 0 } + } + + #[must_use] + pub const fn remaining(&self) -> usize { + self.buf.len() - self.pos + } + + #[must_use] + pub const fn is_empty(&self) -> bool { + self.remaining() == 0 + } + + fn need(&self, n: usize) -> Result<()> { + if self.remaining() < n { + return Err(ParseError::Truncated { + needed: n, + have: self.remaining(), + }); + } + Ok(()) + } + + pub fn u8(&mut self) -> Result { + self.need(1)?; + let v = self.buf[self.pos]; + self.pos += 1; + Ok(v) + } + + pub fn u16(&mut self) -> Result { + self.need(2)?; + let v = u16::from_be_bytes([self.buf[self.pos], self.buf[self.pos + 1]]); + self.pos += 2; + Ok(v) + } + + /// Reads a 24 bit big endian length, the width TLS uses for handshake + /// message bodies and certificate entries. + pub fn u24(&mut self) -> Result { + self.need(3)?; + let v = u32::from_be_bytes([ + 0, + self.buf[self.pos], + self.buf[self.pos + 1], + self.buf[self.pos + 2], + ]); + self.pos += 3; + Ok(v) + } + + pub fn u32(&mut self) -> Result { + self.need(4)?; + let v = u32::from_be_bytes([ + self.buf[self.pos], + self.buf[self.pos + 1], + self.buf[self.pos + 2], + self.buf[self.pos + 3], + ]); + self.pos += 4; + Ok(v) + } + + /// Borrows the next `n` bytes and advances past them. + pub fn take(&mut self, n: usize) -> Result<&'pkt [u8]> { + self.need(n)?; + let slice = &self.buf[self.pos..self.pos + n]; + self.pos += n; + Ok(slice) + } + + /// Reads a one byte length prefix, then borrows that many bytes. + pub fn take_u8_vec(&mut self) -> Result<&'pkt [u8]> { + let len = self.u8()? as usize; + self.take(len) + } + + /// Reads a two byte length prefix, then borrows that many bytes. + pub fn take_u16_vec(&mut self) -> Result<&'pkt [u8]> { + let len = self.u16()? as usize; + self.take(len) + } + + /// Reads a three byte length prefix, then borrows that many bytes. + pub fn take_u24_vec(&mut self) -> Result<&'pkt [u8]> { + let len = self.u24()? as usize; + self.take(len) + } + + /// Returns a sub reader over a two byte length prefixed region. + /// + /// This is the workhorse for nested vectors such as the extensions block, + /// where an outer length governs a region that itself contains a sequence of + /// length prefixed elements. + pub fn sub_u16_vec(&mut self) -> Result> { + let len = self.u16()? as usize; + let slice = self.take(len)?; + Ok(Reader::new(slice)) + } + + /// Returns a sub reader over a three byte length prefixed region, the width + /// the Certificate message uses for its certificate list. + pub fn sub_u24_vec(&mut self) -> Result> { + let len = self.u24()? as usize; + let slice = self.take(len)?; + Ok(Reader::new(slice)) + } +} + +#[cfg(test)] +mod tests { + use super::Reader; + use crate::error::ParseError; + + #[test] + fn reads_widths_in_order() { + let data = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06]; + let mut r = Reader::new(&data); + assert_eq!(r.u8().unwrap(), 0x01); + assert_eq!(r.u16().unwrap(), 0x0203); + assert_eq!(r.u24().unwrap(), 0x0004_0506); + assert!(r.is_empty()); + } + + #[test] + fn short_read_is_an_error_not_a_panic() { + let data = [0x01]; + let mut r = Reader::new(&data); + assert_eq!( + r.u16().unwrap_err(), + ParseError::Truncated { needed: 2, have: 1 } + ); + } + + #[test] + fn length_prefixed_take_respects_bounds() { + let data = [0x03, 0xaa, 0xbb, 0xcc, 0xff]; + let mut r = Reader::new(&data); + assert_eq!(r.take_u8_vec().unwrap(), &[0xaa, 0xbb, 0xcc]); + assert_eq!(r.u8().unwrap(), 0xff); + } + + #[test] + fn sub_vector_isolates_a_region() { + let data = [0x00, 0x02, 0x11, 0x22, 0x33]; + let mut r = Reader::new(&data); + let mut sub = r.sub_u16_vec().unwrap(); + assert_eq!(sub.remaining(), 2); + assert_eq!(sub.u16().unwrap(), 0x1122); + assert_eq!(r.u8().unwrap(), 0x33); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/record.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/record.rs new file mode 100644 index 00000000..6cf0656b --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/record.rs @@ -0,0 +1,179 @@ +// ©AngelaMos | 2026 +// record.rs + +use std::borrow::Cow; + +use smallvec::SmallVec; + +use crate::error::{ParseError, Result}; +use crate::parse::hello::ClientHello; +use crate::parse::reader::Reader; +use crate::registry::content_type; + +/// Reassembles the cleartext handshake flight from a TLS record stream. +/// +/// A handshake message can be split across several TLS records, and several +/// short messages can share one record. This walks the record framing and +/// concatenates the payloads of the handshake records so the caller sees one +/// contiguous handshake byte stream. The common case is a single record holding +/// a single ClientHello, and that case borrows the original bytes with no copy. +/// Only genuinely fragmented flights allocate. +/// +/// Records carrying anything other than handshake data are ignored. In TLS 1.3 +/// the later handshake messages travel inside records typed as application data +/// and are encrypted, so they never reach this function, which is correct: the +/// only handshake bytes we can read in the clear are the first flight. +pub fn handshake_bytes(stream: &[u8]) -> Result> { + let mut segments: SmallVec<[&[u8]; 4]> = SmallVec::new(); + let mut r = Reader::new(stream); + + while r.remaining() >= 5 { + let ctype = r.u8()?; + let _version = r.u16()?; + let payload = r.take_u16_vec()?; + if ctype == content_type::HANDSHAKE { + segments.push(payload); + } else if !segments.is_empty() { + break; + } + } + + match segments.as_slice() { + [] => Err(ParseError::Truncated { + needed: 5, + have: stream.len(), + }), + [only] => Ok(Cow::Borrowed(*only)), + many => { + let mut joined = Vec::with_capacity(many.iter().map(|s| s.len()).sum()); + for seg in many { + joined.extend_from_slice(seg); + } + Ok(Cow::Owned(joined)) + } + } +} + +/// Returns the body of the first handshake message of the requested type. +/// +/// The handshake header is a one byte type and a three byte length. This walks +/// the messages in the reassembled flight and returns the body slice of the +/// first one whose type matches, so the caller never has to reason about the +/// header widths. +pub fn first_handshake_message(handshake: &[u8], want_type: u8) -> Result<&[u8]> { + let mut r = Reader::new(handshake); + while r.remaining() >= 4 { + let msg_type = r.u8()?; + let len = r.u24()? as usize; + let body = r.take(len)?; + if msg_type == want_type { + return Ok(body); + } + } + Err(ParseError::UnexpectedHandshake(want_type)) +} + +/// Returns true when the stream begins with an SSLv2 style ClientHello. +/// +/// SSLv2 framing sets the high bit of the first length byte and places the +/// message type in the first body byte. Type 1 is CLIENT-HELLO. Some old +/// malware opens with this backward compatible hello even when it intends to +/// negotiate TLS, so detecting it keeps the TLS parser from misreading the +/// SSLv2 header as a TLS record. +#[must_use] +pub fn is_sslv2_client_hello(stream: &[u8]) -> bool { + stream.len() >= 3 && (stream[0] & 0x80) != 0 && stream[2] == 1 +} + +/// Parses an SSLv2 style ClientHello into the common ClientHello shape. +/// +/// SSLv2 carries no extensions, supported groups, or point formats, so those +/// stay empty, which matches the community consensus for fingerprinting an +/// SSLv2 hello. Cipher specs are three bytes each. Specs that begin with a zero +/// byte are SSLv3 and TLS cipher suites carried in the backward compatible +/// hello, and those are the values a fingerprint cares about, so they are +/// extracted as their two byte suite numbers. True SSLv2 only specs are +/// counted but cannot be expressed as two byte suites and are skipped. +pub fn parse_sslv2_client_hello(stream: &[u8]) -> Result> { + let mut r = Reader::new(stream); + let len_hi = r.u8()? & 0x7f; + let len_lo = r.u8()?; + let _record_len = (u16::from(len_hi) << 8) | u16::from(len_lo); + + let msg_type = r.u8()?; + if msg_type != 1 { + return Err(ParseError::UnexpectedHandshake(msg_type)); + } + + let legacy_version = r.u16()?; + let cipher_spec_len = r.u16()? as usize; + let session_id_len = r.u16()? as usize; + let challenge_len = r.u16()? as usize; + + let cipher_specs = r.take(cipher_spec_len)?; + let _session_id = r.take(session_id_len)?; + let _challenge = r.take(challenge_len)?; + + let mut cipher_suites = SmallVec::new(); + let mut specs = Reader::new(cipher_specs); + while specs.remaining() >= 3 { + let kind = specs.u8()?; + let suite = specs.u16()?; + if kind == 0 { + cipher_suites.push(suite); + } + } + + Ok(ClientHello { + legacy_version, + cipher_suites, + extensions: SmallVec::new(), + is_sslv2: true, + }) +} + +#[cfg(test)] +mod tests { + use super::{first_handshake_message, handshake_bytes, is_sslv2_client_hello}; + use crate::registry::handshake_type; + + fn record(ctype: u8, payload: &[u8]) -> Vec { + let mut v = vec![ctype, 0x03, 0x03]; + let len = u16::try_from(payload.len()).unwrap(); + v.extend_from_slice(&len.to_be_bytes()); + v.extend_from_slice(payload); + v + } + + #[test] + fn single_record_borrows() { + let stream = record(22, &[0x01, 0x00, 0x00, 0x00]); + let hs = handshake_bytes(&stream).unwrap(); + assert!(matches!(hs, std::borrow::Cow::Borrowed(_))); + assert_eq!(hs.as_ref(), &[0x01, 0x00, 0x00, 0x00]); + } + + #[test] + fn fragmented_records_join() { + let mut stream = record(22, &[0x01, 0x00, 0x00, 0x06, 0xaa, 0xbb]); + stream.extend(record(22, &[0xcc, 0xdd, 0xee, 0xff])); + let hs = handshake_bytes(&stream).unwrap(); + assert!(matches!(hs, std::borrow::Cow::Owned(_))); + assert_eq!(hs.as_ref().len(), 10); + } + + #[test] + fn finds_the_requested_message() { + let hs = [ + 0x02, 0x00, 0x00, 0x01, 0x99, 0x01, 0x00, 0x00, 0x02, 0xaa, 0xbb, + ]; + let body = first_handshake_message(&hs, handshake_type::CLIENT_HELLO).unwrap(); + assert_eq!(body, &[0xaa, 0xbb]); + } + + #[test] + fn sslv2_detection() { + assert!(is_sslv2_client_hello(&[0x80, 0x2e, 0x01, 0x00, 0x02])); + assert!(!is_sslv2_client_hello(&[0x16, 0x03, 0x01, 0x00])); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/decode.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/decode.rs new file mode 100644 index 00000000..6db1b4bd --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/decode.rs @@ -0,0 +1,347 @@ +// ©AngelaMos | 2026 +// decode.rs + +use std::net::{IpAddr, SocketAddr}; + +use etherparse::{EtherType, NetSlice, SlicedPacket, TransportSlice}; +use smallvec::SmallVec; + +use crate::ja4t::TcpFingerprintInput; + +/// LINKTYPE registry numbers this decoder understands. +/// +/// The values come from the tcpdump link layer header type registry. They are +/// redeclared here as plain constants because pcap file readers and live +/// captures both report them as bare integers, and the decoder is the single +/// place that interprets them. +pub mod link_type { + pub const NULL: i32 = 0; + pub const ETHERNET: i32 = 1; + pub const RAW: i32 = 101; + pub const LOOP: i32 = 108; + pub const LINUX_SLL: i32 = 113; + pub const IPV4: i32 = 228; + pub const IPV6: i32 = 229; + pub const LINUX_SLL2: i32 = 276; +} + +/// The BSD null and loopback link headers are four bytes of address family. +const NULL_HEADER_LEN: usize = 4; + +/// The Linux cooked capture v2 header is twenty bytes with the protocol in the +/// first two. +const SLL2_HEADER_LEN: usize = 20; + +/// The TCP kind numbers the JA4T walk extracts values from. +const TCP_OPT_END: u8 = 0; +const TCP_OPT_NOP: u8 = 1; +const TCP_OPT_MSS: u8 = 2; +const TCP_OPT_WSCALE: u8 = 3; +const TCP_OPT_MSS_LEN: u8 = 4; +const TCP_OPT_WSCALE_LEN: u8 = 3; + +/// The TCP flag bits, exactly as byte thirteen of the header carries them. +/// +/// Keeping the flags as the wire bitfield instead of a fistful of bools means +/// the struct mirrors the protocol and reads the byte the packet already +/// holds, rather than rebuilding it from a handful of accessor calls. Adding +/// a flag later is then a constant, not a field. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TcpFlags(u8); + +impl TcpFlags { + pub const FIN: u8 = 0x01; + pub const SYN: u8 = 0x02; + pub const RST: u8 = 0x04; + pub const ACK: u8 = 0x10; + + /// The offset of the flags byte within a TCP header. + const FLAGS_BYTE: usize = 13; + + #[must_use] + pub const fn new(bits: u8) -> Self { + Self(bits) + } + + /// Reads the flags byte from a TCP header slice. + /// + /// A missing byte cannot happen for a slice the decoder hands in, since + /// the transport layer is only present when a full header parsed, but the + /// bounds checked read keeps this honest under direct unit testing. + #[must_use] + fn from_header(header: &[u8]) -> Self { + Self(header.get(Self::FLAGS_BYTE).copied().unwrap_or(0)) + } + + #[must_use] + pub const fn syn(self) -> bool { + self.0 & Self::SYN != 0 + } + + #[must_use] + pub const fn ack(self) -> bool { + self.0 & Self::ACK != 0 + } + + #[must_use] + pub const fn fin(self) -> bool { + self.0 & Self::FIN != 0 + } + + #[must_use] + pub const fn rst(self) -> bool { + self.0 & Self::RST != 0 + } +} + +/// The TCP level facts about one decoded segment. +#[derive(Debug, Clone, Copy)] +pub struct TcpMeta { + pub seq: u32, + pub flags: TcpFlags, + pub window_size: u16, +} + +/// One TCP segment decoded out of a captured frame. +/// +/// Addresses are directional: `src` sent this segment to `dst`. The JA4T +/// input is walked eagerly, but only for SYN packets, because those are the +/// only packets whose options JA4T reads and the walk needs the option bytes +/// that do not outlive the decode. +#[derive(Debug)] +pub struct DecodedSegment<'pkt> { + pub src: SocketAddr, + pub dst: SocketAddr, + pub tcp: TcpMeta, + pub syn_fingerprint: Option, + pub payload: &'pkt [u8], +} + +/// Why a frame produced no segment. The distinction only feeds counters, but +/// the counters are how an operator learns what a capture was made of. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Skip { + UnsupportedLinkType, + NotIp, + NotTcp, + Malformed, +} + +/// Decodes a captured frame down to its TCP segment, if it has one. +/// +/// VLAN tags, including stacked QinQ, are stepped over by etherparse. Frames +/// the decoder does not understand are skipped with a reason rather than +/// failing the capture: a fingerprinting pipeline must shrug off the GRE +/// tunnel, the ARP chatter, and the malformed frame that share every real +/// network with the TLS it cares about. +pub fn decode_frame(link: i32, data: &[u8]) -> Result, Skip> { + let sliced = match link { + link_type::ETHERNET => SlicedPacket::from_ethernet(data), + link_type::LINUX_SLL => SlicedPacket::from_linux_sll(data), + link_type::RAW | link_type::IPV4 | link_type::IPV6 => SlicedPacket::from_ip(data), + link_type::NULL | link_type::LOOP => { + let Some(inner) = data.get(NULL_HEADER_LEN..) else { + return Err(Skip::Malformed); + }; + SlicedPacket::from_ip(inner) + } + link_type::LINUX_SLL2 => { + let Some(proto) = data.first_chunk::<2>() else { + return Err(Skip::Malformed); + }; + let Some(inner) = data.get(SLL2_HEADER_LEN..) else { + return Err(Skip::Malformed); + }; + SlicedPacket::from_ether_type(EtherType(u16::from_be_bytes(*proto)), inner) + } + _ => return Err(Skip::UnsupportedLinkType), + }; + let sliced = sliced.map_err(|_| Skip::Malformed)?; + + let (src_ip, dst_ip): (IpAddr, IpAddr) = match &sliced.net { + Some(NetSlice::Ipv4(v4)) => ( + IpAddr::V4(v4.header().source_addr()), + IpAddr::V4(v4.header().destination_addr()), + ), + Some(NetSlice::Ipv6(v6)) => ( + IpAddr::V6(v6.header().source_addr()), + IpAddr::V6(v6.header().destination_addr()), + ), + Some(NetSlice::Arp(_)) | None => return Err(Skip::NotIp), + }; + + let Some(TransportSlice::Tcp(tcp)) = &sliced.transport else { + return Err(Skip::NotTcp); + }; + + let flags = TcpFlags::from_header(tcp.slice()); + let syn_fingerprint = flags + .syn() + .then(|| tcp_fingerprint_input(tcp.window_size(), tcp.options())); + + Ok(DecodedSegment { + src: SocketAddr::new(src_ip, tcp.source_port()), + dst: SocketAddr::new(dst_ip, tcp.destination_port()), + tcp: TcpMeta { + seq: tcp.sequence_number(), + flags, + window_size: tcp.window_size(), + }, + syn_fingerprint, + payload: tcp.payload(), + }) +} + +/// Walks raw TCP options into the JA4T input. +/// +/// JA4T records every option kind in order, including each NOP and each +/// trailing end of list byte, because the padding pattern is part of how an +/// operating system's stack writes a SYN. The walk is deliberately tolerant: +/// a truncated or nonsense length byte ends the walk after recording the kind +/// it was found on, so a hostile SYN cannot push the parser out of bounds. +pub fn tcp_fingerprint_input(window_size: u16, options: &[u8]) -> TcpFingerprintInput { + let mut kinds: SmallVec<[u8; 8]> = SmallVec::new(); + let mut mss = 0u16; + let mut window_scale = 0u8; + + let mut i = 0; + while i < options.len() { + let kind = options[i]; + kinds.push(kind); + if kind == TCP_OPT_END || kind == TCP_OPT_NOP { + i += 1; + continue; + } + let Some(&len) = options.get(i + 1) else { + break; + }; + if len < 2 { + break; + } + let Some(body) = options.get(i + 2..i + usize::from(len)) else { + break; + }; + if kind == TCP_OPT_MSS && len == TCP_OPT_MSS_LEN { + if let Some(value) = body.first_chunk::<2>() { + mss = u16::from_be_bytes(*value); + } + } + if kind == TCP_OPT_WSCALE && len == TCP_OPT_WSCALE_LEN { + if let Some(&value) = body.first() { + window_scale = value; + } + } + i += usize::from(len); + } + + TcpFingerprintInput { + window_size, + option_kinds: kinds, + mss, + window_scale, + } +} + +#[cfg(test)] +mod tests { + use super::{Skip, TcpFlags, decode_frame, link_type, tcp_fingerprint_input}; + use crate::ja4t::ja4t; + use etherparse::PacketBuilder; + + fn tcp_frame(payload: &[u8]) -> Vec { + let builder = PacketBuilder::ethernet2([1; 6], [2; 6]) + .ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64) + .tcp(40000, 443, 1000, 64240); + let mut out = Vec::with_capacity(builder.size(payload.len())); + builder.write(&mut out, payload).unwrap(); + out + } + + #[test] + fn flags_byte_decodes_to_the_right_bits() { + let syn = TcpFlags::from_header(&[0u8; 14]); + assert!(!syn.syn()); + + let mut header = [0u8; 20]; + header[13] = TcpFlags::SYN | TcpFlags::ACK; + let flags = TcpFlags::from_header(&header); + assert!(flags.syn() && flags.ack()); + assert!(!flags.fin() && !flags.rst()); + + assert!(!TcpFlags::from_header(&[]).syn()); + } + + #[test] + fn decodes_an_ethernet_tcp_frame() { + let frame = tcp_frame(b"hello"); + let seg = decode_frame(link_type::ETHERNET, &frame).unwrap(); + assert_eq!(seg.src.to_string(), "10.0.0.1:40000"); + assert_eq!(seg.dst.to_string(), "10.0.0.2:443"); + assert_eq!(seg.tcp.seq, 1000); + assert_eq!(seg.payload, b"hello"); + } + + #[test] + fn decodes_a_vlan_tagged_frame() { + let builder = PacketBuilder::ethernet2([1; 6], [2; 6]) + .single_vlan(etherparse::VlanId::try_new(7).unwrap()) + .ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64) + .tcp(40000, 443, 1, 64240); + let mut frame = Vec::with_capacity(builder.size(0)); + builder.write(&mut frame, &[]).unwrap(); + + let seg = decode_frame(link_type::ETHERNET, &frame).unwrap(); + assert_eq!(seg.dst.port(), 443); + } + + #[test] + fn non_tcp_and_garbage_are_skips_not_panics() { + let builder = PacketBuilder::ethernet2([1; 6], [2; 6]) + .ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64) + .udp(5000, 53); + let mut udp = Vec::with_capacity(builder.size(4)); + builder.write(&mut udp, &[0xde, 0xad, 0xbe, 0xef]).unwrap(); + + assert!(matches!( + decode_frame(link_type::ETHERNET, &udp), + Err(Skip::NotTcp) + )); + assert!(matches!( + decode_frame(link_type::ETHERNET, &[0x01, 0x02]), + Err(Skip::Malformed) + )); + assert!(matches!( + decode_frame(147, &udp), + Err(Skip::UnsupportedLinkType) + )); + } + + #[test] + fn ja4t_walk_reproduces_the_linux_default_vector() { + let options = [ + 0x02, 0x04, 0x05, 0xb4, 0x01, 0x03, 0x03, 0x08, 0x01, 0x01, 0x04, 0x02, + ]; + let input = tcp_fingerprint_input(64240, &options); + assert_eq!(ja4t(&input), "64240_2-1-3-1-1-4_1460_8"); + } + + #[test] + fn ja4t_walk_counts_trailing_end_of_list_padding() { + let options = [ + 0x02, 0x04, 0x05, 0x42, 0x01, 0x03, 0x03, 0x06, 0x01, 0x01, 0x08, 0x0a, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x02, 0x00, 0x00, + ]; + let input = tcp_fingerprint_input(65535, &options); + assert_eq!(ja4t(&input), "65535_2-1-3-1-1-8-4-0-0_1346_6"); + } + + #[test] + fn ja4t_walk_survives_truncated_options() { + let input = tcp_fingerprint_input(1024, &[0x02, 0x04, 0x05]); + assert_eq!(input.option_kinds.as_slice(), &[0x02]); + assert_eq!(input.mss, 0); + + let zero_len = tcp_fingerprint_input(1024, &[0x05, 0x00, 0x02]); + assert_eq!(zero_len.option_kinds.as_slice(), &[0x05]); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/event.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/event.rs new file mode 100644 index 00000000..70d2c9fe --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/event.rs @@ -0,0 +1,135 @@ +// ©AngelaMos | 2026 +// event.rs + +use std::fmt; +use std::net::SocketAddr; + +use serde::Serialize; + +use crate::fingerprint::{Ja3, Ja4Family}; + +/// A fingerprint produced by one direction of one flow, without addressing. +/// +/// The protocol layer emits these; the pipeline wraps them with the flow's +/// addresses and timestamp to make a [`FingerprintEvent`]. Keeping the two +/// layers apart means the protocol extractor can be tested with bare byte +/// streams, no packets required. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum StreamEvent { + ClientHello { + ja3: Ja3, + ja3_raw: String, + ja4: Ja4Family, + sni: Option, + alpn: Option, + }, + ServerHello { + ja3s: Ja3, + ja3s_raw: String, + ja4s: Ja4Family, + }, + Certificate { + ja4x: String, + }, + HttpRequest { + ja4h: Ja4Family, + method: String, + host: Option, + }, + TcpSyn { + ja4t: String, + }, + TcpSynAck { + ja4ts: String, + }, +} + +/// One fingerprint observation, addressed and timestamped. +/// +/// `src` is always the party that sent the fingerprinted bytes: the client +/// for a ClientHello or SYN, the server for a ServerHello or certificate. +#[derive(Debug, Clone, Serialize)] +pub struct FingerprintEvent { + pub ts_nanos: u64, + pub src: SocketAddr, + pub dst: SocketAddr, + #[serde(flatten)] + pub event: StreamEvent, +} + +impl fmt::Display for FingerprintEvent { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let secs = self.ts_nanos / 1_000_000_000; + let millis = self.ts_nanos % 1_000_000_000 / 1_000_000; + write!(f, "{secs}.{millis:03} {} -> {} ", self.src, self.dst)?; + match &self.event { + StreamEvent::ClientHello { + ja3, + ja4, + sni, + alpn, + .. + } => { + write!(f, "client_hello ja4={} ja3={ja3}", ja4.hash)?; + if let Some(sni) = sni { + write!(f, " sni={sni}")?; + } + if let Some(alpn) = alpn { + write!(f, " alpn={alpn}")?; + } + Ok(()) + } + StreamEvent::ServerHello { ja3s, ja4s, .. } => { + write!(f, "server_hello ja4s={} ja3s={ja3s}", ja4s.hash) + } + StreamEvent::Certificate { ja4x } => write!(f, "certificate ja4x={ja4x}"), + StreamEvent::HttpRequest { ja4h, method, host } => { + write!(f, "http_request ja4h={} method={method}", ja4h.hash)?; + if let Some(host) = host { + write!(f, " host={host}")?; + } + Ok(()) + } + StreamEvent::TcpSyn { ja4t } => write!(f, "tcp_syn ja4t={ja4t}"), + StreamEvent::TcpSynAck { ja4ts } => write!(f, "tcp_syn_ack ja4ts={ja4ts}"), + } + } +} + +#[cfg(test)] +mod tests { + use super::{FingerprintEvent, StreamEvent}; + + #[test] + fn json_shape_is_tagged_and_flat() { + let event = FingerprintEvent { + ts_nanos: 1_500_000_000, + src: "10.0.0.1:40000".parse().unwrap(), + dst: "10.0.0.2:443".parse().unwrap(), + event: StreamEvent::Certificate { + ja4x: "7d5dbb3783b4_ba7ce0880c07_7bf9a7bf7029".into(), + }, + }; + let json = serde_json::to_value(&event).unwrap(); + assert_eq!(json["kind"], "certificate"); + assert_eq!(json["ja4x"], "7d5dbb3783b4_ba7ce0880c07_7bf9a7bf7029"); + assert_eq!(json["src"], "10.0.0.1:40000"); + } + + #[test] + fn display_is_one_greppable_line() { + let event = FingerprintEvent { + ts_nanos: 1_234_000_000, + src: "10.0.0.1:40000".parse().unwrap(), + dst: "10.0.0.2:443".parse().unwrap(), + event: StreamEvent::TcpSyn { + ja4t: "64240_2-1-3-1-1-4_1460_8".into(), + }, + }; + assert_eq!( + event.to_string(), + "1.234 10.0.0.1:40000 -> 10.0.0.2:443 tcp_syn ja4t=64240_2-1-3-1-1-4_1460_8" + ); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/flow.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/flow.rs new file mode 100644 index 00000000..a2e28ae7 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/flow.rs @@ -0,0 +1,430 @@ +// ©AngelaMos | 2026 +// flow.rs + +use std::collections::BTreeMap; +use std::net::SocketAddr; + +/// A bidirectional flow identity. +/// +/// The two endpoints are stored in sorted order so that a packet and its reply +/// hash to the same key. Which endpoint is the client is a separate question, +/// answered by who sent the SYN or, failing that, who spoke a ClientHello, and +/// it is deliberately not baked into the key: captures routinely start in the +/// middle of connections, and a key that guessed wrong would split one +/// conversation into two. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct FlowKey { + pub lo: SocketAddr, + pub hi: SocketAddr, +} + +/// Which endpoint of a [`FlowKey`] sent a given segment. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Direction { + FromLo, + FromHi, +} + +impl Direction { + #[must_use] + pub const fn index(self) -> usize { + match self { + Direction::FromLo => 0, + Direction::FromHi => 1, + } + } + + /// The source and destination addresses of traffic flowing this way. + #[must_use] + pub const fn addresses(self, key: &FlowKey) -> (SocketAddr, SocketAddr) { + match self { + Direction::FromLo => (key.lo, key.hi), + Direction::FromHi => (key.hi, key.lo), + } + } +} + +impl FlowKey { + /// Normalizes a directional (source, destination) pair into a key plus the + /// direction the packet travelled. + #[must_use] + pub fn from_pair(src: SocketAddr, dst: SocketAddr) -> (Self, Direction) { + if src <= dst { + (Self { lo: src, hi: dst }, Direction::FromLo) + } else { + (Self { lo: dst, hi: src }, Direction::FromHi) + } + } +} + +/// The midpoint of the sequence space. In TCP serial arithmetic an offset at +/// or beyond this point is read as the segment sitting behind the anchor, not +/// absurdly far ahead of it. +const HALF_SERIAL_SPACE: u32 = 0x8000_0000; + +/// Resource limits for one reassembled direction of one flow. +#[derive(Debug, Clone, Copy)] +pub struct ReassemblyLimits { + /// Most contiguous bytes kept. Everything a passive fingerprinter reads + /// sits in the first kilobytes of a stream, so this is a cap on patience, + /// not on correctness. + pub max_assembled_bytes: usize, + /// Most bytes parked in the out of order buffer. + pub max_pending_bytes: usize, + /// Most segments parked in the out of order buffer. + pub max_pending_segments: usize, +} + +/// What [`StreamReassembler::push`] did with a segment. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PushOutcome { + /// The contiguous stream grew; the protocol layer should look again. + Grew, + /// Nothing new: a duplicate, pure overlap, or empty segment. + Unchanged, + /// The segment was parked out of order for later. + Parked, + /// The segment fell outside the window this reassembler is willing to + /// track, or a buffer limit was hit, and it was dropped. + Dropped, +} + +/// Reassembles one direction of a TCP stream into contiguous bytes. +/// +/// This is the piece most toy fingerprinting tools skip, and skipping it is +/// why they miss handshakes: a ClientHello, and even more so a certificate +/// chain, regularly spans several segments, and those segments arrive +/// reordered on any path with packet loss. The reassembler anchors at the +/// sequence number the SYN names or, on a flow whose start the capture +/// missed, at the first segment it sees. Everything else is a relative +/// offset from that anchor in wrapping serial arithmetic: in order segments +/// append to one contiguous buffer, out of order segments park in a map +/// keyed by offset until the gap before them fills. +/// +/// Data from before the anchor on a SYN-less flow is gone; a streaming +/// engine cannot retroactively prepend, and accepting that loss explicitly +/// is what Suricata does for midstream pickup too. A segment that straddles +/// the anchor is trimmed to its useful part rather than discarded. +/// +/// Overlaps resolve first write wins: bytes already accepted are never +/// rewritten by a later segment. A passive observer cannot know which copy +/// the receiver kept, and the CVE-2018-6794 capture in the test corpus exists +/// precisely because inconsistent overlap handling let attackers show an IDS +/// a different stream than the one the victim read. First write wins is one +/// deterministic, documented answer. +#[derive(Debug)] +pub struct StreamReassembler { + limits: ReassemblyLimits, + anchor: Option, + assembled: Vec, + pending: BTreeMap>, + pending_bytes: usize, + released: bool, + capped: bool, +} + +impl StreamReassembler { + #[must_use] + pub fn new(limits: ReassemblyLimits) -> Self { + Self { + limits, + anchor: None, + assembled: Vec::new(), + pending: BTreeMap::new(), + pending_bytes: 0, + released: false, + capped: false, + } + } + + /// The contiguous bytes assembled so far, from the anchor onward. + #[must_use] + pub fn data(&self) -> &[u8] { + &self.assembled + } + + /// Pins the stream start, used when a SYN reveals the true initial + /// sequence number before any data arrives. Later anchors are ignored. + pub fn anchor(&mut self, seq: u32) { + if self.anchor.is_none() { + self.anchor = Some(seq); + } + } + + /// Drops every buffer and refuses all future data. + /// + /// Called once the protocol layer has what it needs, or knows it never + /// will. This is what keeps memory flat when a capture contains long + /// lived flows: the flow entry stays, the payload buffers do not. + pub fn release(&mut self) { + self.assembled = Vec::new(); + self.pending = BTreeMap::new(); + self.pending_bytes = 0; + self.released = true; + } + + #[must_use] + pub fn released(&self) -> bool { + self.released + } + + /// True when the assembled cap was hit and the tail of the stream is gone. + #[must_use] + pub fn capped(&self) -> bool { + self.capped + } + + /// Offers one segment to the stream. + pub fn push(&mut self, seq: u32, payload: &[u8]) -> PushOutcome { + if self.released || payload.is_empty() { + return PushOutcome::Unchanged; + } + if self.capped { + return PushOutcome::Dropped; + } + let anchor = *self.anchor.get_or_insert(seq); + + let offset = seq.wrapping_sub(anchor); + if offset >= HALF_SERIAL_SPACE { + let stale = offset.wrapping_neg() as usize; + if stale >= payload.len() { + return PushOutcome::Unchanged; + } + return self.push(anchor, &payload[stale..]); + } + let window_end = self + .limits + .max_assembled_bytes + .saturating_add(self.limits.max_pending_bytes); + if offset as usize > window_end { + return PushOutcome::Dropped; + } + + let assembled_len = self.assembled.len(); + if (offset as usize) < assembled_len { + let overlap = assembled_len - offset as usize; + if overlap >= payload.len() { + return PushOutcome::Unchanged; + } + return self.append_in_order(&payload[overlap..]); + } + if offset as usize == assembled_len { + return self.append_in_order(payload); + } + + if self.pending.len() >= self.limits.max_pending_segments + || self.pending_bytes.saturating_add(payload.len()) > self.limits.max_pending_bytes + { + return PushOutcome::Dropped; + } + match self.pending.entry(offset) { + std::collections::btree_map::Entry::Occupied(existing) => { + if existing.get().len() >= payload.len() { + return PushOutcome::Unchanged; + } + self.pending_bytes += payload.len() - existing.get().len(); + *existing.into_mut() = payload.to_vec(); + } + std::collections::btree_map::Entry::Vacant(slot) => { + self.pending_bytes += payload.len(); + slot.insert(payload.to_vec()); + } + } + PushOutcome::Parked + } + + fn append_in_order(&mut self, payload: &[u8]) -> PushOutcome { + let room = self + .limits + .max_assembled_bytes + .saturating_sub(self.assembled.len()); + if room == 0 { + self.mark_capped(); + return PushOutcome::Dropped; + } + let take = payload.len().min(room); + self.assembled.extend_from_slice(&payload[..take]); + if take < payload.len() { + self.mark_capped(); + } else { + self.drain_pending(); + } + PushOutcome::Grew + } + + /// Splices parked segments onto the contiguous buffer while they touch it. + fn drain_pending(&mut self) { + while let Some(entry) = self.pending.first_entry() { + let offset = *entry.key() as usize; + if offset > self.assembled.len() { + break; + } + let segment = entry.remove(); + self.pending_bytes -= segment.len(); + let overlap = self.assembled.len() - offset; + if overlap >= segment.len() { + continue; + } + let room = self + .limits + .max_assembled_bytes + .saturating_sub(self.assembled.len()); + let take = (segment.len() - overlap).min(room); + self.assembled + .extend_from_slice(&segment[overlap..overlap + take]); + if take < segment.len() - overlap { + self.mark_capped(); + return; + } + } + } + + /// Once the assembled cap is hit nothing later can ever become contiguous, + /// so the parked segments are garbage. Drop them and refuse new data, but + /// keep the assembled prefix: it is still a valid stream head and whatever + /// the protocol layer already read from it stands. + fn mark_capped(&mut self) { + self.capped = true; + self.pending = BTreeMap::new(); + self.pending_bytes = 0; + } +} + +#[cfg(test)] +mod tests { + use super::{FlowKey, PushOutcome, ReassemblyLimits, StreamReassembler}; + + fn limits() -> ReassemblyLimits { + ReassemblyLimits { + max_assembled_bytes: 64, + max_pending_bytes: 64, + max_pending_segments: 4, + } + } + + fn reasm() -> StreamReassembler { + StreamReassembler::new(limits()) + } + + #[test] + fn both_directions_share_one_key() { + let a: std::net::SocketAddr = "10.0.0.1:40000".parse().unwrap(); + let b: std::net::SocketAddr = "10.0.0.2:443".parse().unwrap(); + let (forward, fwd_dir) = FlowKey::from_pair(a, b); + let (reverse, rev_dir) = FlowKey::from_pair(b, a); + assert_eq!(forward, reverse); + assert_ne!(fwd_dir, rev_dir); + assert_eq!(fwd_dir.addresses(&forward), (a, b)); + assert_eq!(rev_dir.addresses(&reverse), (b, a)); + } + + #[test] + fn in_order_segments_concatenate() { + let mut r = reasm(); + assert_eq!(r.push(100, b"hell"), PushOutcome::Grew); + assert_eq!(r.push(104, b"o"), PushOutcome::Grew); + assert_eq!(r.data(), b"hello"); + } + + #[test] + fn out_of_order_segments_wait_for_the_gap() { + let mut r = reasm(); + assert_eq!(r.push(100, b"hell"), PushOutcome::Grew); + assert_eq!(r.push(107, b"orld"), PushOutcome::Parked); + assert_eq!(r.data(), b"hell"); + assert_eq!(r.push(104, b"o w"), PushOutcome::Grew); + assert_eq!(r.data(), b"hello world"); + } + + #[test] + fn anchor_from_syn_orders_data_arriving_backwards() { + let mut r = reasm(); + r.anchor(1000); + assert_eq!(r.push(1004, b"data"), PushOutcome::Parked); + assert_eq!(r.push(1000, b"more"), PushOutcome::Grew); + assert_eq!(r.data(), b"moredata"); + } + + #[test] + fn retransmissions_change_nothing() { + let mut r = reasm(); + r.push(100, b"abcdef"); + assert_eq!(r.push(100, b"abcdef"), PushOutcome::Unchanged); + assert_eq!(r.push(102, b"cd"), PushOutcome::Unchanged); + assert_eq!(r.data(), b"abcdef"); + } + + #[test] + fn overlapping_segment_keeps_the_first_write() { + let mut r = reasm(); + r.push(100, b"abcdef"); + assert_eq!(r.push(103, b"XXXghi"), PushOutcome::Grew); + assert_eq!(r.data(), b"abcdefghi"); + } + + #[test] + fn parked_overlap_keeps_the_first_write_too() { + let mut r = reasm(); + r.anchor(100); + assert_eq!(r.push(104, b"efgh"), PushOutcome::Parked); + assert_eq!(r.push(100, b"abcdEFG"), PushOutcome::Grew); + assert_eq!(r.data(), b"abcdEFGh"); + } + + #[test] + fn sequence_numbers_wrap_around_zero() { + let mut r = reasm(); + let anchor = u32::MAX - 1; + assert_eq!(r.push(anchor, b"ab"), PushOutcome::Grew); + assert_eq!(r.push(0, b"cd"), PushOutcome::Grew); + assert_eq!(r.push(2, b"ef"), PushOutcome::Grew); + assert_eq!(r.data(), b"abcdef"); + } + + #[test] + fn stale_pre_anchor_data_is_ignored_and_far_future_dropped() { + let mut r = reasm(); + r.push(1000, b"ab"); + assert_eq!(r.push(990, b"old"), PushOutcome::Unchanged); + assert_eq!(r.push(100_000, b"far"), PushOutcome::Dropped); + assert_eq!(r.data(), b"ab"); + } + + #[test] + fn segment_straddling_the_anchor_is_trimmed_not_lost() { + let mut r = reasm(); + r.anchor(1000); + assert_eq!(r.push(996, b"oldNEW"), PushOutcome::Grew); + assert_eq!(r.data(), b"EW"); + } + + #[test] + fn assembled_cap_truncates_but_keeps_the_prefix() { + let mut r = reasm(); + let big = vec![0x41u8; 100]; + assert_eq!(r.push(0, &big), PushOutcome::Grew); + assert_eq!(r.data().len(), limits().max_assembled_bytes); + assert_eq!(r.push(100, b"more"), PushOutcome::Dropped); + } + + #[test] + fn pending_limits_drop_excess_segments() { + let mut r = reasm(); + r.anchor(0); + assert_eq!(r.push(10, b"a"), PushOutcome::Parked); + assert_eq!(r.push(20, b"b"), PushOutcome::Parked); + assert_eq!(r.push(30, b"c"), PushOutcome::Parked); + assert_eq!(r.push(40, b"d"), PushOutcome::Parked); + assert_eq!(r.push(50, b"e"), PushOutcome::Dropped); + } + + #[test] + fn release_drops_buffers_and_refuses_data() { + let mut r = reasm(); + r.push(0, b"abc"); + r.release(); + assert!(r.released()); + assert_eq!(r.data(), b""); + assert_eq!(r.push(3, b"def"), PushOutcome::Unchanged); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/mod.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/mod.rs new file mode 100644 index 00000000..fc98eba6 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/mod.rs @@ -0,0 +1,433 @@ +// ©AngelaMos | 2026 +// mod.rs + +//! The passive fingerprinting pipeline: frames in, fingerprint events out. +//! +//! The stages are deliberately separable. A [`PacketSource`] yields raw link +//! layer frames from a capture file or, later, a live interface. The decoder +//! strips the frame down to a TCP segment. The flow table reassembles each +//! direction of each conversation into a contiguous byte stream, surviving +//! reordering, retransmission, and overlap. The protocol layer watches each +//! stream until it recognizes a TLS flight or an HTTP request head and emits +//! fingerprints. Nothing in here touches a network interface, so the whole +//! pipeline runs byte exact in tests against vendored captures. + +pub mod decode; +pub mod event; +pub mod flow; +pub mod source; +pub mod tls; + +use std::collections::HashMap; + +use serde::Serialize; + +use crate::ja4t::ja4t; +use crate::pipeline::decode::{Skip, decode_frame}; +use crate::pipeline::event::{FingerprintEvent, StreamEvent}; +use crate::pipeline::flow::{FlowKey, PushOutcome, ReassemblyLimits, StreamReassembler}; +use crate::pipeline::source::{PacketSource, RawFrame, SourceError}; +use crate::pipeline::tls::StreamProtocol; + +/// Tuning knobs for the pipeline. +/// +/// The defaults are sized for handshake harvesting: generous enough that a +/// fat certificate chain or a multi segment ClientHello always fits, small +/// enough that an adversarial capture cannot turn the flow table into a +/// memory bomb. +#[derive(Debug, Clone, Copy)] +pub struct PipelineConfig { + /// Flows tracked at once before the table sheds its oldest entries. + pub max_flows: usize, + /// A flow untouched for this long is eligible for eviction. + pub idle_timeout_nanos: u64, + /// Contiguous bytes kept per direction. + pub max_assembled_bytes: usize, + /// Out of order bytes parked per direction. + pub max_pending_bytes: usize, + /// Out of order segments parked per direction. + pub max_pending_segments: usize, +} + +impl PipelineConfig { + pub const DEFAULT_MAX_FLOWS: usize = 65_536; + pub const DEFAULT_IDLE_TIMEOUT_NANOS: u64 = 60 * 1_000_000_000; + pub const DEFAULT_MAX_ASSEMBLED_BYTES: usize = 256 * 1024; + pub const DEFAULT_MAX_PENDING_BYTES: usize = 256 * 1024; + pub const DEFAULT_MAX_PENDING_SEGMENTS: usize = 128; + + fn limits(&self) -> ReassemblyLimits { + ReassemblyLimits { + max_assembled_bytes: self.max_assembled_bytes, + max_pending_bytes: self.max_pending_bytes, + max_pending_segments: self.max_pending_segments, + } + } +} + +impl Default for PipelineConfig { + fn default() -> Self { + Self { + max_flows: Self::DEFAULT_MAX_FLOWS, + idle_timeout_nanos: Self::DEFAULT_IDLE_TIMEOUT_NANOS, + max_assembled_bytes: Self::DEFAULT_MAX_ASSEMBLED_BYTES, + max_pending_bytes: Self::DEFAULT_MAX_PENDING_BYTES, + max_pending_segments: Self::DEFAULT_MAX_PENDING_SEGMENTS, + } + } +} + +/// What the pipeline saw, for the operator and for the miss rate honesty +/// check: a fingerprinting tool that cannot say what it failed to read is a +/// tool whose silence gets mistaken for absence. +#[derive(Debug, Default, Clone, Copy, Serialize)] +pub struct Counters { + pub frames: u64, + pub bytes: u64, + pub tcp_segments: u64, + pub skipped_unsupported_link_type: u64, + pub skipped_not_ip: u64, + pub skipped_not_tcp: u64, + pub skipped_malformed: u64, + pub flows_created: u64, + pub flows_evicted_idle: u64, + pub flows_evicted_pressure: u64, + pub segments_dropped: u64, + pub events: u64, + pub streams_capped: u64, + pub unfinished_tls_streams: u64, +} + +/// One direction of one tracked flow. +struct StreamHalf { + reassembler: StreamReassembler, + protocol: StreamProtocol, + syn_fingerprint_emitted: bool, +} + +impl StreamHalf { + fn new(limits: ReassemblyLimits) -> Self { + Self { + reassembler: StreamReassembler::new(limits), + protocol: StreamProtocol::Undecided, + syn_fingerprint_emitted: false, + } + } +} + +struct FlowState { + halves: [StreamHalf; 2], + last_seen_nanos: u64, +} + +impl FlowState { + fn new(limits: ReassemblyLimits) -> Self { + Self { + halves: [StreamHalf::new(limits), StreamHalf::new(limits)], + last_seen_nanos: 0, + } + } + + fn finished(&self) -> bool { + self.halves.iter().all(|h| h.protocol.finished()) + } +} + +/// The passive fingerprinting engine. +/// +/// Feed it frames, take events out through the sink closure. The pipeline is +/// synchronous and single threaded by design: one pipeline owns its flow +/// table outright, and running one per worker beats sharing a locked table +/// between workers. +pub struct Pipeline { + config: PipelineConfig, + flows: HashMap, + counters: Counters, +} + +impl Pipeline { + #[must_use] + pub fn new(config: PipelineConfig) -> Self { + Self { + config, + flows: HashMap::new(), + counters: Counters::default(), + } + } + + #[must_use] + pub fn counters(&self) -> &Counters { + &self.counters + } + + /// Drains a source through the pipeline, sending every event to `sink`. + pub fn run( + &mut self, + source: &mut S, + mut sink: impl FnMut(FingerprintEvent), + ) -> Result<(), SourceError> { + while let Some(frame) = source.next_frame()? { + self.feed(&frame, &mut sink); + } + self.finish(); + Ok(()) + } + + /// Processes one captured frame. + pub fn feed(&mut self, frame: &RawFrame<'_>, sink: &mut impl FnMut(FingerprintEvent)) { + self.counters.frames += 1; + self.counters.bytes += frame.data.len() as u64; + + let segment = match decode_frame(frame.link_type, frame.data) { + Ok(segment) => segment, + Err(skip) => { + match skip { + Skip::UnsupportedLinkType => { + self.counters.skipped_unsupported_link_type += 1; + } + Skip::NotIp => self.counters.skipped_not_ip += 1, + Skip::NotTcp => self.counters.skipped_not_tcp += 1, + Skip::Malformed => self.counters.skipped_malformed += 1, + } + return; + } + }; + self.counters.tcp_segments += 1; + + let (key, direction) = FlowKey::from_pair(segment.src, segment.dst); + if !self.flows.contains_key(&key) { + if self.flows.len() >= self.config.max_flows { + self.evict(frame.ts_nanos); + } + self.flows.insert(key, FlowState::new(self.config.limits())); + self.counters.flows_created += 1; + } + let Some(flow) = self.flows.get_mut(&key) else { + return; + }; + flow.last_seen_nanos = flow.last_seen_nanos.max(frame.ts_nanos); + + let (src, dst) = direction.addresses(&key); + let half = &mut flow.halves[direction.index()]; + + if !half.syn_fingerprint_emitted { + if let Some(input) = &segment.syn_fingerprint { + half.syn_fingerprint_emitted = true; + let fingerprint = ja4t(input); + let event = if segment.tcp.flags.ack() { + StreamEvent::TcpSynAck { ja4ts: fingerprint } + } else { + StreamEvent::TcpSyn { ja4t: fingerprint } + }; + self.counters.events += 1; + sink(FingerprintEvent { + ts_nanos: frame.ts_nanos, + src, + dst, + event, + }); + } + } + + if segment.tcp.flags.syn() { + half.reassembler.anchor(segment.tcp.seq.wrapping_add(1)); + } + + let payload_seq = if segment.tcp.flags.syn() { + segment.tcp.seq.wrapping_add(1) + } else { + segment.tcp.seq + }; + let outcome = half.reassembler.push(payload_seq, segment.payload); + if outcome == PushOutcome::Dropped { + self.counters.segments_dropped += 1; + } + + if outcome == PushOutcome::Grew { + let mut emitted = 0u64; + tls::advance(&mut half.protocol, half.reassembler.data(), &mut |event| { + emitted += 1; + sink(FingerprintEvent { + ts_nanos: frame.ts_nanos, + src, + dst, + event, + }); + }); + self.counters.events += emitted; + if half.protocol.finished() && !half.reassembler.released() { + half.reassembler.release(); + } + } + } + + /// Settles the books at end of capture. + /// + /// Streams that were recognized as TLS but never produced a complete + /// handshake message are counted: each one is a handshake the capture + /// clipped, which is exactly the number an operator needs before trusting + /// an absence of fingerprints. + pub fn finish(&mut self) { + for flow in self.flows.values() { + for half in &flow.halves { + if half.protocol.unfinished_tls() { + self.counters.unfinished_tls_streams += 1; + } + if half.reassembler.capped() { + self.counters.streams_capped += 1; + } + } + } + self.flows.clear(); + } + + /// Sheds flows when the table is full: everything idle past the timeout + /// or fully harvested goes, and if nothing qualifies, the single stalest + /// flow goes, so the table never refuses a brand new conversation in + /// favor of a dead one. + fn evict(&mut self, now_nanos: u64) { + let timeout = self.config.idle_timeout_nanos; + let idle: Vec = self + .flows + .iter() + .filter(|(_, flow)| { + now_nanos.saturating_sub(flow.last_seen_nanos) > timeout || flow.finished() + }) + .map(|(key, _)| *key) + .collect(); + + if idle.is_empty() { + let stalest = self + .flows + .iter() + .min_by_key(|(_, flow)| flow.last_seen_nanos) + .map(|(key, _)| *key); + if let Some(key) = stalest { + self.drop_flow(key); + self.counters.flows_evicted_pressure += 1; + } + return; + } + + for key in idle { + self.drop_flow(key); + self.counters.flows_evicted_idle += 1; + } + } + + fn drop_flow(&mut self, key: FlowKey) { + if let Some(flow) = self.flows.remove(&key) { + for half in &flow.halves { + if half.protocol.unfinished_tls() { + self.counters.unfinished_tls_streams += 1; + } + if half.reassembler.capped() { + self.counters.streams_capped += 1; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::{Pipeline, PipelineConfig}; + use crate::pipeline::event::FingerprintEvent; + use crate::pipeline::source::RawFrame; + use etherparse::PacketBuilder; + + fn tcp_frame(src: ([u8; 4], u16), dst: ([u8; 4], u16), seq: u32, payload: &[u8]) -> Vec { + let builder = PacketBuilder::ethernet2([1; 6], [2; 6]) + .ipv4(src.0, dst.0, 64) + .tcp(src.1, dst.1, seq, 64240); + let mut out = Vec::with_capacity(builder.size(payload.len())); + builder.write(&mut out, payload).unwrap(); + out + } + + fn feed_all(pipeline: &mut Pipeline, frames: &[Vec]) -> Vec { + let mut events = Vec::new(); + for (i, data) in frames.iter().enumerate() { + let frame = RawFrame { + ts_nanos: u64::try_from(i).unwrap() * 1_000_000, + link_type: 1, + data, + }; + pipeline.feed(&frame, &mut |e| events.push(e)); + } + events + } + + #[test] + fn http_request_split_across_segments_fingerprints_once() { + let client = ([10, 0, 0, 1], 40000); + let server = ([10, 0, 0, 2], 80); + let request = b"GET / HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n"; + let (a, b) = request.split_at(20); + + let frames = vec![ + tcp_frame(client, server, 1000, a), + tcp_frame(client, server, 1000 + u32::try_from(a.len()).unwrap(), b), + ]; + + let mut pipeline = Pipeline::new(PipelineConfig::default()); + let events = feed_all(&mut pipeline, &frames); + assert_eq!(events.len(), 1); + assert_eq!(events[0].src.to_string(), "10.0.0.1:40000"); + assert_eq!(pipeline.counters().tcp_segments, 2); + } + + #[test] + fn out_of_order_delivery_after_a_syn_still_fingerprints() { + let client = ([10, 0, 0, 1], 40001); + let server = ([10, 0, 0, 2], 80); + let request = b"GET / HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n"; + let (a, b) = request.split_at(20); + + let syn = { + let builder = PacketBuilder::ethernet2([1; 6], [2; 6]) + .ipv4(client.0, server.0, 64) + .tcp(client.1, server.1, 999, 64240) + .syn(); + let mut out = Vec::with_capacity(builder.size(0)); + builder.write(&mut out, &[]).unwrap(); + out + }; + + let frames = vec![ + syn, + tcp_frame(client, server, 1000 + u32::try_from(a.len()).unwrap(), b), + tcp_frame(client, server, 1000, a), + ]; + + let mut pipeline = Pipeline::new(PipelineConfig::default()); + let events = feed_all(&mut pipeline, &frames); + assert_eq!(events.len(), 2); + assert!(events[0].to_string().contains("tcp_syn ja4t=")); + assert!(events[1].to_string().contains("http_request")); + } + + #[test] + fn pressure_eviction_keeps_the_table_bounded() { + let config = PipelineConfig { + max_flows: 4, + ..PipelineConfig::default() + }; + let mut pipeline = Pipeline::new(config); + + let mut frames = Vec::new(); + for i in 0..8u16 { + let port = 40000 + i; + frames.push(tcp_frame( + ([10, 0, 0, 1], port), + ([10, 0, 0, 2], 80), + 1, + b"x", + )); + } + feed_all(&mut pipeline, &frames); + + assert_eq!(pipeline.counters().flows_created, 8); + assert!(pipeline.counters().flows_evicted_pressure >= 4); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/source.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/source.rs new file mode 100644 index 00000000..8a05413f --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/source.rs @@ -0,0 +1,362 @@ +// ©AngelaMos | 2026 +// source.rs + +use std::fs::File; +use std::io::Read; +use std::path::Path; + +use pcap_parser::traits::PcapReaderIterator; +use pcap_parser::{Block, PcapBlockOwned, PcapError, PcapHeader, create_reader}; +use thiserror::Error; + +/// How many bytes of buffer the file reader starts with. +/// +/// A single capture block must fit in the buffer. Offload features such as TSO +/// can put frames far larger than an MTU into a capture, so the buffer starts +/// generous and can still grow up to [`MAX_BUFFER_CAPACITY`] if a bigger block +/// appears. +const INITIAL_BUFFER_CAPACITY: usize = 1024 * 1024; + +/// The ceiling for buffer growth. A block larger than this is treated as a +/// malformed capture rather than a reason to exhaust memory. +const MAX_BUFFER_CAPACITY: usize = 64 * 1024 * 1024; + +/// Timestamp units per second when a capture does not say otherwise. +/// +/// Both the legacy pcap format and the pcapng default are microsecond +/// resolution. +const DEFAULT_UNITS_PER_SECOND: u64 = 1_000_000; + +const NANOS_PER_SECOND: u64 = 1_000_000_000; +const NANOS_PER_MICRO: u64 = 1_000; + +/// Errors produced while reading frames from a capture source. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum SourceError { + #[error("failed to read capture: {0}")] + Io(#[from] std::io::Error), + + #[error("not a pcap or pcapng capture")] + NotACapture, + + #[error("capture block exceeds the {MAX_BUFFER_CAPACITY} byte buffer ceiling")] + BlockTooLarge, + + #[error("malformed capture: {0}")] + Malformed(String), +} + +/// One link layer frame as captured, with the metadata needed to decode it. +#[derive(Debug, Clone, Copy)] +pub struct RawFrame<'src> { + /// Capture timestamp in nanoseconds since the epoch. Zero when the capture + /// format carries no timestamp for this frame. + pub ts_nanos: u64, + /// The link layer type, using the tcpdump LINKTYPE registry numbers. + pub link_type: i32, + pub data: &'src [u8], +} + +/// A source of captured frames. +/// +/// The trait is a lending iterator: each frame borrows from the source and is +/// only valid until the next call. That shape fits both file readers, which +/// hand out windows into an internal buffer, and live captures, which hand out +/// the kernel's buffer. A consumer that needs to keep a frame longer copies +/// it, and that decision stays visible at the call site. +pub trait PacketSource { + /// Returns the next frame, or `None` when the source is exhausted. + fn next_frame(&mut self) -> Result>, SourceError>; +} + +/// Per interface metadata from a pcapng interface description block. +#[derive(Debug, Clone, Copy)] +struct InterfaceInfo { + link_type: i32, + units_per_second: u64, + ts_offset_seconds: i64, +} + +/// Everything in the source except the parser, split out so the borrow of the +/// parser's buffer held by a block and the mutable borrow needed to stage a +/// frame land on different fields. +#[derive(Default)] +struct SourceState { + interfaces: Vec, + legacy: Option, + legacy_nanos: bool, + frame: Vec, + frame_ts_nanos: u64, + frame_link_type: i32, +} + +impl SourceState { + /// Copies a frame out of the parser's buffer so the borrow on the parser + /// can end before the block is consumed. + fn stage(&mut self, ts_nanos: u64, link_type: i32, data: &[u8]) { + self.frame.clear(); + self.frame.extend_from_slice(data); + self.frame_ts_nanos = ts_nanos; + self.frame_link_type = link_type; + } + + fn handle_legacy_header(&mut self, header: &PcapHeader) { + self.legacy = Some(InterfaceInfo { + link_type: header.network.0, + units_per_second: DEFAULT_UNITS_PER_SECOND, + ts_offset_seconds: 0, + }); + self.legacy_nanos = header.is_nanosecond_precision(); + } + + /// Stages a packet block. Returns false for metadata blocks. + fn handle_block(&mut self, block: &PcapBlockOwned<'_>) -> bool { + match block { + PcapBlockOwned::LegacyHeader(header) => { + self.handle_legacy_header(header); + false + } + PcapBlockOwned::Legacy(frame) => { + let Some(meta) = self.legacy else { + return false; + }; + let fraction = if self.legacy_nanos { + u64::from(frame.ts_usec) + } else { + u64::from(frame.ts_usec) * NANOS_PER_MICRO + }; + let ts = u64::from(frame.ts_sec) + .saturating_mul(NANOS_PER_SECOND) + .saturating_add(fraction); + let len = frame.data.len().min(frame.caplen as usize); + self.stage(ts, meta.link_type, &frame.data[..len]); + true + } + PcapBlockOwned::NG(Block::SectionHeader(_)) => { + self.interfaces.clear(); + false + } + PcapBlockOwned::NG(Block::InterfaceDescription(idb)) => { + self.interfaces.push(InterfaceInfo { + link_type: idb.linktype.0, + units_per_second: idb.ts_resolution().unwrap_or(DEFAULT_UNITS_PER_SECOND), + ts_offset_seconds: idb.ts_offset(), + }); + false + } + PcapBlockOwned::NG(Block::EnhancedPacket(epb)) => { + let Some(meta) = self.interfaces.get(epb.if_id as usize).copied() else { + return false; + }; + let units = (u64::from(epb.ts_high) << 32) | u64::from(epb.ts_low); + let ts = scale_to_nanos(units, meta.units_per_second, meta.ts_offset_seconds); + let len = epb.data.len().min(epb.caplen as usize); + self.stage(ts, meta.link_type, &epb.data[..len]); + true + } + PcapBlockOwned::NG(Block::SimplePacket(spb)) => { + let Some(meta) = self.interfaces.first().copied() else { + return false; + }; + let len = spb.data.len().min(spb.origlen as usize); + self.stage(0, meta.link_type, &spb.data[..len]); + true + } + PcapBlockOwned::NG(_) => false, + } + } +} + +/// Reads frames from a pcap or pcapng file. +/// +/// The two formats are probed automatically. pcapng is handled with its full +/// generality: every interface carries its own link type and timestamp +/// resolution, multiple sections reset the interface list, and metadata blocks +/// such as name resolution and decryption secrets are skipped rather than +/// treated as packets. A truncated final packet, the signature of a capture +/// that was stopped rather than closed, ends iteration cleanly and is reported +/// through [`PcapFileSource::truncated`] instead of failing the whole file. +pub struct PcapFileSource { + reader: Box, + state: SourceState, + buffer_capacity: usize, + truncated: bool, + finished: bool, +} + +impl PcapFileSource { + /// Opens a capture file from a path. + pub fn open(path: impl AsRef) -> Result { + Self::from_reader(File::open(path)?) + } + + /// Builds a source from any byte reader holding pcap or pcapng data. + pub fn from_reader(reader: impl Read + 'static) -> Result { + let reader = create_reader(INITIAL_BUFFER_CAPACITY, reader).map_err(|e| match e { + PcapError::HeaderNotRecognized | PcapError::Eof => SourceError::NotACapture, + PcapError::ReadError => SourceError::Io(std::io::Error::other("read failed")), + other => SourceError::Malformed(other.to_string()), + })?; + Ok(Self { + reader, + state: SourceState::default(), + buffer_capacity: INITIAL_BUFFER_CAPACITY, + truncated: false, + finished: false, + }) + } + + /// Returns true when the file ended in the middle of a block. + pub fn truncated(&self) -> bool { + self.truncated + } +} + +impl PacketSource for PcapFileSource { + fn next_frame(&mut self) -> Result>, SourceError> { + if self.finished { + return Ok(None); + } + loop { + let staged = match self.reader.next() { + Ok((offset, block)) => { + let staged = self.state.handle_block(&block); + self.reader.consume(offset); + staged + } + Err(PcapError::Eof) => { + self.finished = true; + return Ok(None); + } + Err(PcapError::UnexpectedEof) => { + self.finished = true; + self.truncated = true; + return Ok(None); + } + Err(PcapError::Incomplete(_)) => { + self.reader + .refill() + .map_err(|e| SourceError::Malformed(e.to_string()))?; + continue; + } + Err(PcapError::BufferTooSmall) => { + let grown = self.buffer_capacity.saturating_mul(2); + if grown > MAX_BUFFER_CAPACITY || !self.reader.grow(grown) { + self.finished = true; + return Err(SourceError::BlockTooLarge); + } + self.buffer_capacity = grown; + continue; + } + Err(PcapError::ReadError) => { + self.finished = true; + return Err(SourceError::Io(std::io::Error::other("read failed"))); + } + Err( + e @ (PcapError::HeaderNotRecognized + | PcapError::NomError(..) + | PcapError::OwnedNomError(..)), + ) => { + self.finished = true; + return Err(SourceError::Malformed(e.to_string())); + } + }; + if staged { + return Ok(Some(RawFrame { + ts_nanos: self.state.frame_ts_nanos, + link_type: self.state.frame_link_type, + data: &self.state.frame, + })); + } + } + } +} + +/// Converts a timestamp in interface units to nanoseconds since the epoch. +/// +/// The arithmetic runs in 128 bits so the conversion stays exact for every +/// resolution pcapng can express, including nanosecond counts that already +/// fill most of a u64. +fn scale_to_nanos(units: u64, units_per_second: u64, offset_seconds: i64) -> u64 { + if units_per_second == 0 { + return 0; + } + let nanos = u128::from(units) * u128::from(NANOS_PER_SECOND) / u128::from(units_per_second); + let offset = i128::from(offset_seconds) * i128::from(NANOS_PER_SECOND); + u64::try_from( + i128::try_from(nanos) + .unwrap_or(i128::MAX) + .saturating_add(offset), + ) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::{PacketSource, PcapFileSource, SourceError, scale_to_nanos}; + + fn legacy_pcap(frames: &[&[u8]]) -> Vec { + let mut v = Vec::new(); + v.extend_from_slice(&0xa1b2_c3d4_u32.to_le_bytes()); + v.extend_from_slice(&2u16.to_le_bytes()); + v.extend_from_slice(&4u16.to_le_bytes()); + v.extend_from_slice(&0i32.to_le_bytes()); + v.extend_from_slice(&0u32.to_le_bytes()); + v.extend_from_slice(&65535u32.to_le_bytes()); + v.extend_from_slice(&1u32.to_le_bytes()); + for (i, frame) in frames.iter().enumerate() { + let len = u32::try_from(frame.len()).unwrap(); + v.extend_from_slice(&u32::try_from(i + 1).unwrap().to_le_bytes()); + v.extend_from_slice(&500_000u32.to_le_bytes()); + v.extend_from_slice(&len.to_le_bytes()); + v.extend_from_slice(&len.to_le_bytes()); + v.extend_from_slice(frame); + } + v + } + + #[test] + fn reads_legacy_frames_with_timestamps() { + let data = legacy_pcap(&[&[0xaa; 14], &[0xbb; 20]]); + let mut source = PcapFileSource::from_reader(std::io::Cursor::new(data)).unwrap(); + + let one = source.next_frame().unwrap().unwrap(); + assert_eq!(one.link_type, 1); + assert_eq!(one.ts_nanos, 1_500_000_000); + assert_eq!(one.data.len(), 14); + + let two = source.next_frame().unwrap().unwrap(); + assert_eq!(two.data, &[0xbb; 20]); + + assert!(source.next_frame().unwrap().is_none()); + assert!(!source.truncated()); + } + + #[test] + fn truncated_final_frame_ends_cleanly() { + let mut data = legacy_pcap(&[&[0xaa; 14], &[0xbb; 20]]); + data.truncate(data.len() - 5); + let mut source = PcapFileSource::from_reader(std::io::Cursor::new(data)).unwrap(); + + assert!(source.next_frame().unwrap().is_some()); + assert!(source.next_frame().unwrap().is_none()); + assert!(source.truncated()); + assert!(source.next_frame().unwrap().is_none()); + } + + #[test] + fn garbage_is_not_a_capture() { + let err = PcapFileSource::from_reader(std::io::Cursor::new(vec![0x55; 64])) + .err() + .unwrap(); + assert!(matches!(err, SourceError::NotACapture)); + } + + #[test] + fn timestamp_scaling_is_exact_for_common_resolutions() { + assert_eq!(scale_to_nanos(1_500_000, 1_000_000, 0), 1_500_000_000); + assert_eq!(scale_to_nanos(7, 1_000_000_000, 0), 7); + assert_eq!(scale_to_nanos(1, 1, 1), 2_000_000_000); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/tls.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/tls.rs new file mode 100644 index 00000000..9f242408 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/tls.rs @@ -0,0 +1,401 @@ +// ©AngelaMos | 2026 +// tls.rs + +use std::borrow::Cow; + +use crate::ja3::{ja3, ja3_string, ja3s, ja3s_string}; +use crate::ja4::{Transport, ja4, ja4s}; +use crate::ja4h::{ja4h, parse_http_request}; +use crate::ja4x::ja4x; +use crate::parse::reader::Reader; +use crate::parse::{ + certificate_der_list, is_sslv2_client_hello, parse_client_hello, parse_server_hello, + parse_sslv2_client_hello, +}; +use crate::pipeline::event::StreamEvent; +use crate::registry::{content_type, handshake_type, version}; + +/// How many bytes the sniffer needs before it gives up on classifying a +/// stream. Every protocol this pipeline recognizes shows its hand within the +/// first few bytes; eight covers the longest HTTP method prefix. +const SNIFF_DECISION_LEN: usize = 8; + +/// The HTTP methods the sniffer accepts as the start of a cleartext request. +const HTTP_METHOD_PREFIXES: [&[u8]; 9] = [ + b"GET ", + b"POST ", + b"PUT ", + b"HEAD ", + b"DELETE ", + b"OPTIONS ", + b"PATCH ", + b"TRACE ", + b"CONNECT ", +]; + +/// A stream whose HTTP request head has not finished inside this many bytes +/// is not worth waiting on. +const HTTP_HEAD_CAP: usize = 8 * 1024; + +const HTTP_HEAD_TERMINATOR: &[u8] = b"\r\n\r\n"; + +/// A TLS record payload cannot exceed 2^14 plus expansion; RFC 8446 allows +/// 255 bytes of expansion on top of the 16384 byte plaintext limit. A length +/// beyond that means the stream is not actually TLS record framing. +const MAX_TLS_RECORD_LEN: usize = 16384 + 255; + +/// What one direction of a flow is, as far as the protocol layer can tell. +#[derive(Debug)] +pub enum StreamProtocol { + /// Not enough bytes yet to say. + Undecided, + /// TLS record framing; the cleartext first flight is being extracted. + Tls(TlsFlight), + /// A cleartext HTTP/1.x request head is being accumulated. + Http, + /// Recognized and fully harvested; the stream needs no more buffering. + Done, + /// Unrecognized or unparseable; the stream is ignored. + Ignored, +} + +impl StreamProtocol { + /// True when this direction will never produce another event, which is + /// the signal to drop its reassembly buffers. + #[must_use] + pub fn finished(&self) -> bool { + matches!(self, StreamProtocol::Done | StreamProtocol::Ignored) + } + + /// True when the stream was recognized as TLS but the capture ended + /// before a complete hello could be read. Feeds the miss rate counter + /// that tells an operator their capture is clipping handshakes. + #[must_use] + pub fn unfinished_tls(&self) -> bool { + match self { + StreamProtocol::Tls(flight) => !flight.saw_any_message, + _ => false, + } + } +} + +/// Incremental extraction state for one direction's cleartext TLS flight. +/// +/// Only what must survive between walks lives here. A ClientHello ends its +/// direction immediately, so it needs no flag; a ServerHello does not, since +/// a TLS 1.2 Certificate may still be in flight behind it, so the emission +/// guard for it persists. +#[derive(Debug, Default)] +pub struct TlsFlight { + emitted_server_hello: bool, + saw_any_message: bool, +} + +/// Drives protocol detection and extraction over one direction of a stream. +/// +/// `stream` is always the full contiguous bytes from the start of the +/// direction; the extractor re-walks them on each call. That sounds wasteful +/// and is not: the walk is linear over at most the reassembly cap, the +/// interesting messages sit in the first packets, and re-walking from the +/// start is what makes a message that arrives split across three segments +/// parse correctly with no incremental parser state to get wrong. +/// +/// Returns events through `sink` and updates `proto` in place. +pub fn advance(proto: &mut StreamProtocol, stream: &[u8], sink: &mut impl FnMut(StreamEvent)) { + if matches!(proto, StreamProtocol::Undecided) { + *proto = sniff(stream); + } + match proto { + StreamProtocol::Undecided | StreamProtocol::Done | StreamProtocol::Ignored => {} + StreamProtocol::Tls(_) => advance_tls(proto, stream, sink), + StreamProtocol::Http => advance_http(proto, stream, sink), + } +} + +/// Classifies the first bytes of a stream. +fn sniff(stream: &[u8]) -> StreamProtocol { + if stream.len() >= 3 { + if stream[0] == content_type::HANDSHAKE && stream[1] == 0x03 && stream[2] <= 0x04 { + return StreamProtocol::Tls(TlsFlight::default()); + } + if is_sslv2_client_hello(stream) { + return StreamProtocol::Tls(TlsFlight::default()); + } + } + if stream.len() >= SNIFF_DECISION_LEN { + if HTTP_METHOD_PREFIXES.iter().any(|m| stream.starts_with(m)) { + return StreamProtocol::Http; + } + return StreamProtocol::Ignored; + } + StreamProtocol::Undecided +} + +fn advance_http(proto: &mut StreamProtocol, stream: &[u8], sink: &mut impl FnMut(StreamEvent)) { + let head_end = stream + .windows(HTTP_HEAD_TERMINATOR.len()) + .position(|w| w == HTTP_HEAD_TERMINATOR); + let Some(head_end) = head_end else { + if stream.len() > HTTP_HEAD_CAP { + *proto = StreamProtocol::Ignored; + } + return; + }; + + let head = &stream[..head_end + HTTP_HEAD_TERMINATOR.len()]; + if let Some(request) = parse_http_request(head) { + let host = request + .headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("host")) + .map(|(_, value)| value.clone()); + sink(StreamEvent::HttpRequest { + ja4h: ja4h(&request), + method: request.method.clone(), + host, + }); + *proto = StreamProtocol::Done; + } else { + *proto = StreamProtocol::Ignored; + } +} + +fn advance_tls(proto: &mut StreamProtocol, stream: &[u8], sink: &mut impl FnMut(StreamEvent)) { + if is_sslv2_client_hello(stream) { + advance_sslv2(proto, stream, sink); + return; + } + + let Some(flight_bytes) = collect_flight(stream) else { + *proto = StreamProtocol::Ignored; + return; + }; + + let StreamProtocol::Tls(flight) = proto else { + return; + }; + let mut done = walk_messages(flight, flight_bytes.handshake.as_ref(), sink); + if flight_bytes.flight_closed && !done { + done = true; + } + if done { + *proto = StreamProtocol::Done; + } +} + +fn advance_sslv2(proto: &mut StreamProtocol, stream: &[u8], sink: &mut impl FnMut(StreamEvent)) { + match parse_sslv2_client_hello(stream) { + Ok(hello) => { + sink(StreamEvent::ClientHello { + ja3: ja3(&hello), + ja3_raw: ja3_string(&hello), + ja4: ja4(&hello, Transport::Tcp), + sni: None, + alpn: None, + }); + *proto = StreamProtocol::Done; + } + Err(crate::error::ParseError::Truncated { .. }) => {} + Err(_) => *proto = StreamProtocol::Ignored, + } +} + +struct FlightBytes<'stream> { + handshake: Cow<'stream, [u8]>, + /// True when a non handshake record followed the handshake records, which + /// in cleartext TLS means the readable part of the flight is over. + flight_closed: bool, +} + +/// Collects the payloads of the leading complete handshake records. +/// +/// Unlike the strict reassembled flight walker in the parse module, this +/// tolerates a trailing partial record, because the stream is still growing. +/// Returns `None` when the bytes stop looking like TLS record framing at all. +fn collect_flight(stream: &[u8]) -> Option> { + let mut segments: Vec<&[u8]> = Vec::new(); + let mut r = Reader::new(stream); + let mut flight_closed = false; + + while r.remaining() >= 5 { + let Ok(ctype) = r.u8() else { break }; + let Ok(record_version) = r.u16() else { break }; + if !matches!( + ctype, + content_type::CHANGE_CIPHER_SPEC + | content_type::ALERT + | content_type::HANDSHAKE + | content_type::APPLICATION_DATA + ) { + return None; + } + if (record_version & 0xff00) != 0x0300 || (record_version & 0x00ff) > 0x04 { + return None; + } + let Ok(declared) = r.u16() else { break }; + if declared as usize > MAX_TLS_RECORD_LEN { + return None; + } + let Ok(payload) = r.take(declared as usize) else { + break; + }; + if ctype == content_type::HANDSHAKE { + segments.push(payload); + } else if !segments.is_empty() { + flight_closed = true; + break; + } + } + + let handshake = match segments.as_slice() { + [] => Cow::Borrowed(&[][..]), + [only] => Cow::Borrowed(*only), + many => { + let mut joined = Vec::with_capacity(many.iter().map(|s| s.len()).sum()); + for seg in many { + joined.extend_from_slice(seg); + } + Cow::Owned(joined) + } + }; + Some(FlightBytes { + handshake, + flight_closed, + }) +} + +/// Walks the complete handshake messages in the flight, emitting fingerprints +/// for the ones that carry them. Returns true when this direction has yielded +/// everything it ever will. +fn walk_messages( + flight: &mut TlsFlight, + handshake: &[u8], + sink: &mut impl FnMut(StreamEvent), +) -> bool { + let mut r = Reader::new(handshake); + while r.remaining() >= 4 { + let Ok(msg_type) = r.u8() else { break }; + let Ok(len) = r.u24() else { break }; + let Ok(body) = r.take(len as usize) else { + break; + }; + flight.saw_any_message = true; + + match msg_type { + handshake_type::CLIENT_HELLO => { + if let Ok(hello) = parse_client_hello(body) { + sink(StreamEvent::ClientHello { + ja3: ja3(&hello), + ja3_raw: ja3_string(&hello), + ja4: ja4(&hello, Transport::Tcp), + sni: hello.server_name().map(str::to_owned), + alpn: hello + .alpn_protocols() + .first() + .map(|p| String::from_utf8_lossy(p).into_owned()), + }); + } + return true; + } + handshake_type::SERVER_HELLO => { + if !flight.emitted_server_hello { + if let Ok(hello) = parse_server_hello(body) { + flight.emitted_server_hello = true; + let negotiated_tls13 = hello.selected_version() == version::TLS_1_3; + sink(StreamEvent::ServerHello { + ja3s: ja3s(&hello), + ja3s_raw: ja3s_string(&hello), + ja4s: ja4s(&hello, Transport::Tcp), + }); + if negotiated_tls13 { + return true; + } + } + } + } + handshake_type::CERTIFICATE => { + if let Ok(certs) = certificate_der_list(body) { + for cert in certs { + if let Ok(fingerprint) = ja4x(cert) { + sink(StreamEvent::Certificate { ja4x: fingerprint }); + } + } + } + return true; + } + _ => {} + } + } + false +} + +#[cfg(test)] +mod tests { + use super::{StreamProtocol, advance, sniff}; + use crate::pipeline::event::StreamEvent; + + fn record(payload: &[u8]) -> Vec { + let mut v = vec![0x16, 0x03, 0x01]; + v.extend_from_slice(&u16::try_from(payload.len()).unwrap().to_be_bytes()); + v.extend_from_slice(payload); + v + } + + #[test] + fn sniffs_tls_http_and_garbage() { + assert!(matches!( + sniff(&[0x16, 0x03, 0x01, 0x00, 0x05]), + StreamProtocol::Tls(_) + )); + assert!(matches!(sniff(b"GET / HTTP/1.1\r\n"), StreamProtocol::Http)); + assert!(matches!( + sniff(b"SSH-2.0-OpenSSH_9.7"), + StreamProtocol::Ignored + )); + assert!(matches!(sniff(b"GE"), StreamProtocol::Undecided)); + } + + #[test] + fn http_request_yields_ja4h_once_head_completes() { + let mut proto = StreamProtocol::Undecided; + let mut events = Vec::new(); + + let partial = b"GET / HTTP/1.1\r\nHost: example.com\r\n"; + advance(&mut proto, partial, &mut |e| events.push(e)); + assert!(events.is_empty()); + assert!(matches!(proto, StreamProtocol::Http)); + + let full = b"GET / HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n"; + advance(&mut proto, full, &mut |e| events.push(e)); + assert_eq!(events.len(), 1); + assert!(matches!( + &events[0], + StreamEvent::HttpRequest { method, host: Some(h), .. } + if method == "GET" && h == "example.com" + )); + assert!(proto.finished()); + } + + #[test] + fn partial_tls_record_waits_for_more_bytes() { + let mut proto = StreamProtocol::Undecided; + let mut events = Vec::new(); + + let full = record(&[0x01, 0x00, 0x00, 0x02, 0xaa, 0xbb]); + advance(&mut proto, &full[..7], &mut |e| events.push(e)); + assert!(events.is_empty()); + assert!(matches!(proto, StreamProtocol::Tls(_))); + assert!(!proto.finished()); + } + + #[test] + fn nonsense_record_length_poisons_the_stream() { + let mut proto = StreamProtocol::Undecided; + let mut events = Vec::new(); + + let stream = [0x16, 0x03, 0x01, 0xff, 0xff, 0x00, 0x00, 0x00]; + advance(&mut proto, &stream, &mut |e| events.push(e)); + assert!(matches!(proto, StreamProtocol::Ignored)); + assert!(events.is_empty()); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/registry.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/registry.rs new file mode 100644 index 00000000..bd6af809 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/registry.rs @@ -0,0 +1,91 @@ +// ©AngelaMos | 2026 +// registry.rs + +//! Named constants for the slices of the TLS and DTLS registries that the +//! fingerprint algorithms reference by number. +//! +//! Wire values are kept as raw `u16` everywhere in this crate rather than being +//! decoded into a closed enum. A fingerprinting engine must preserve cipher and +//! extension values it has never seen, because a future RFC value or a vendor +//! specific extension is exactly the kind of detail that makes one client +//! distinguishable from another. Decoding into a closed enum would collapse all +//! unknown values into a single bucket and corrupt the fingerprint. + +/// TLS and DTLS record content types (the first byte of a record). +pub mod content_type { + pub const CHANGE_CIPHER_SPEC: u8 = 20; + pub const ALERT: u8 = 21; + pub const HANDSHAKE: u8 = 22; + pub const APPLICATION_DATA: u8 = 23; +} + +/// Handshake message types (the first byte of a handshake message body). +pub mod handshake_type { + pub const CLIENT_HELLO: u8 = 1; + pub const SERVER_HELLO: u8 = 2; + pub const CERTIFICATE: u8 = 11; +} + +/// Extension type numbers that the fingerprint algorithms treat specially. +pub mod extension { + pub const SERVER_NAME: u16 = 0x0000; + pub const SUPPORTED_GROUPS: u16 = 0x000a; + pub const EC_POINT_FORMATS: u16 = 0x000b; + pub const SIGNATURE_ALGORITHMS: u16 = 0x000d; + pub const ALPN: u16 = 0x0010; + pub const SUPPORTED_VERSIONS: u16 = 0x002b; +} + +/// Legacy and negotiated protocol version words. +pub mod version { + pub const SSL_2_0: u16 = 0x0002; + pub const SSL_3_0: u16 = 0x0300; + pub const TLS_1_0: u16 = 0x0301; + pub const TLS_1_1: u16 = 0x0302; + pub const TLS_1_2: u16 = 0x0303; + pub const TLS_1_3: u16 = 0x0304; + pub const DTLS_1_0: u16 = 0xfeff; + pub const DTLS_1_2: u16 = 0xfefd; + pub const DTLS_1_3: u16 = 0xfefc; +} + +/// Maps a protocol version word to the two character JA4 version code. +/// +/// The mapping is taken verbatim from the FoxIO JA4 specification. Unknown +/// words collapse to `00`, which is the specified fallback. DTLS words are +/// included even though the Python reference omits them, because the published +/// specification lists them and the Wireshark and Rust references honor them. +#[must_use] +pub fn ja4_version_code(word: u16) -> &'static str { + match word { + version::TLS_1_3 => "13", + version::TLS_1_2 => "12", + version::TLS_1_1 => "11", + version::TLS_1_0 => "10", + version::SSL_3_0 => "s3", + version::SSL_2_0 => "s2", + version::DTLS_1_0 => "d1", + version::DTLS_1_2 => "d2", + version::DTLS_1_3 => "d3", + _ => "00", + } +} + +#[cfg(test)] +mod tests { + use super::ja4_version_code; + use super::version; + + #[test] + fn known_versions_map() { + assert_eq!(ja4_version_code(version::TLS_1_3), "13"); + assert_eq!(ja4_version_code(version::TLS_1_2), "12"); + assert_eq!(ja4_version_code(version::SSL_3_0), "s3"); + assert_eq!(ja4_version_code(version::DTLS_1_2), "d2"); + } + + #[test] + fn unknown_version_falls_back() { + assert_eq!(ja4_version_code(0x7f1d), "00"); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/common/mod.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/common/mod.rs new file mode 100644 index 00000000..3bdf1939 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/common/mod.rs @@ -0,0 +1,161 @@ +// ©AngelaMos | 2026 +// mod.rs + +//! Builders that assemble TLS handshake bytes for tests. +//! +//! Hand assembling wire bytes keeps the fingerprint tests honest. Rather than +//! trusting a higher level library to produce a ClientHello, the tests state the +//! exact cipher list, extension list, and extension bodies, then assert that the +//! parser and the fingerprint algorithms read them back the way the +//! specifications require. +//! +//! Each integration test binary links this module independently, so a builder +//! method that one binary does not call reads as dead code there even though +//! another binary uses it. The allow keeps that shared infrastructure honest +//! without scattering per method annotations. +#![allow(dead_code)] + +/// Assembles a ClientHello handshake message body. +#[derive(Default)] +pub struct ClientHelloBuilder { + legacy_version: u16, + cipher_suites: Vec, + extensions: Vec<(u16, Vec)>, +} + +impl ClientHelloBuilder { + pub fn new() -> Self { + Self { + legacy_version: 0x0303, + ..Self::default() + } + } + + pub fn legacy_version(mut self, v: u16) -> Self { + self.legacy_version = v; + self + } + + pub fn ciphers(mut self, suites: &[u16]) -> Self { + self.cipher_suites = suites.to_vec(); + self + } + + pub fn extension(mut self, ext_type: u16, data: Vec) -> Self { + self.extensions.push((ext_type, data)); + self + } + + /// Adds a server name indication extension for the given host. + pub fn sni(self, host: &str) -> Self { + let host = host.as_bytes(); + let mut entry = vec![0u8]; + push_u16_vec(&mut entry, host); + let mut data = Vec::new(); + push_u16_vec(&mut data, &entry); + self.extension(0x0000, data) + } + + /// Adds a supported groups extension. + pub fn supported_groups(self, groups: &[u16]) -> Self { + let mut list = Vec::new(); + for g in groups { + list.extend_from_slice(&g.to_be_bytes()); + } + let mut data = Vec::new(); + push_u16_vec(&mut data, &list); + self.extension(0x000a, data) + } + + /// Adds an elliptic curve point formats extension. + pub fn ec_point_formats(self, formats: &[u8]) -> Self { + let mut data = Vec::new(); + push_u8_vec(&mut data, formats); + self.extension(0x000b, data) + } + + /// Adds a signature algorithms extension. + pub fn signature_algorithms(self, algs: &[u16]) -> Self { + let mut list = Vec::new(); + for a in algs { + list.extend_from_slice(&a.to_be_bytes()); + } + let mut data = Vec::new(); + push_u16_vec(&mut data, &list); + self.extension(0x000d, data) + } + + /// Adds an ALPN extension advertising the given protocols. + pub fn alpn(self, protos: &[&[u8]]) -> Self { + let mut list = Vec::new(); + for p in protos { + push_u8_vec(&mut list, p); + } + let mut data = Vec::new(); + push_u16_vec(&mut data, &list); + self.extension(0x0010, data) + } + + /// Adds a supported versions extension. + pub fn supported_versions(self, versions: &[u16]) -> Self { + let mut list = Vec::new(); + for v in versions { + list.extend_from_slice(&v.to_be_bytes()); + } + let mut data = Vec::new(); + push_u8_vec(&mut data, &list); + self.extension(0x002b, data) + } + + /// Returns the handshake message body, the input to `parse_client_hello`. + pub fn build_body(&self) -> Vec { + let mut body = Vec::new(); + body.extend_from_slice(&self.legacy_version.to_be_bytes()); + body.extend_from_slice(&[0u8; 32]); + body.push(0); + + let mut ciphers = Vec::new(); + for c in &self.cipher_suites { + ciphers.extend_from_slice(&c.to_be_bytes()); + } + push_u16_vec(&mut body, &ciphers); + + push_u8_vec(&mut body, &[0]); + + let mut exts = Vec::new(); + for (ext_type, data) in &self.extensions { + exts.extend_from_slice(&ext_type.to_be_bytes()); + push_u16_vec(&mut exts, data); + } + push_u16_vec(&mut body, &exts); + body + } + + /// Wraps the handshake body in a handshake header and a TLS record so the + /// result is a complete record stream. + pub fn build_record(&self) -> Vec { + let body = self.build_body(); + let mut msg = vec![1u8]; + push_u24_vec(&mut msg, &body); + + let mut record = vec![22u8, 0x03, 0x01]; + push_u16_vec(&mut record, &msg); + record + } +} + +fn push_u8_vec(out: &mut Vec, data: &[u8]) { + out.push(u8::try_from(data.len()).unwrap()); + out.extend_from_slice(data); +} + +fn push_u16_vec(out: &mut Vec, data: &[u8]) { + out.extend_from_slice(&u16::try_from(data.len()).unwrap().to_be_bytes()); + out.extend_from_slice(data); +} + +fn push_u24_vec(out: &mut Vec, data: &[u8]) { + let len = u32::try_from(data.len()).unwrap(); + out.extend_from_slice(&len.to_be_bytes()[1..]); + out.extend_from_slice(data); +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/ja3.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/ja3.rs new file mode 100644 index 00000000..2e734a78 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/ja3.rs @@ -0,0 +1,55 @@ +// ©AngelaMos | 2026 +// ja3.rs + +mod common; + +use common::ClientHelloBuilder; +use tlsfp_core::ja3::{ja3, ja3_string}; +use tlsfp_core::parse::parse_client_hello; + +/// Rebuilds the exact ClientHello that produces the first published Salesforce +/// JA3 vector, then checks both the pre hash string and the digest end to end. +/// The SNI body content is arbitrary because JA3 records only that extension +/// type zero was present, not its value. +#[test] +fn end_to_end_matches_salesforce_vector_one() { + let body = ClientHelloBuilder::new() + .legacy_version(0x0301) + .ciphers(&[47, 53, 5, 10, 49161, 49162, 49171, 49172, 50, 56, 19, 4]) + .extension(0x0000, vec![0x00, 0x00]) + .supported_groups(&[23, 24, 25]) + .ec_point_formats(&[0]) + .build_body(); + + let ch = parse_client_hello(&body).unwrap(); + assert_eq!( + ja3_string(&ch), + "769,47-53-5-10-49161-49162-49171-49172-50-56-19-4,0-10-11,23-24-25,0" + ); + assert_eq!(ja3(&ch).to_string(), "ada70206e40642a3e4461f35503241d5"); +} + +/// A ClientHello with GREASE in ciphers, extensions, and supported groups must +/// produce the same JA3 as the same hello without GREASE, because GREASE is +/// stripped from every list field before hashing. +#[test] +fn grease_does_not_change_the_fingerprint() { + let clean = ClientHelloBuilder::new() + .legacy_version(0x0303) + .ciphers(&[0x1301, 0x1302]) + .supported_groups(&[0x001d, 0x0017]) + .ec_point_formats(&[0]) + .build_body(); + + let greasy = ClientHelloBuilder::new() + .legacy_version(0x0303) + .ciphers(&[0x0a0a, 0x1301, 0x1302]) + .extension(0x1a1a, vec![]) + .supported_groups(&[0x2a2a, 0x001d, 0x0017]) + .ec_point_formats(&[0]) + .build_body(); + + let clean = parse_client_hello(&clean).unwrap(); + let greasy = parse_client_hello(&greasy).unwrap(); + assert_eq!(ja3(&clean), ja3(&greasy)); +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/ja4.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/ja4.rs new file mode 100644 index 00000000..d1520269 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/ja4.rs @@ -0,0 +1,88 @@ +// ©AngelaMos | 2026 +// ja4.rs + +mod common; + +use common::ClientHelloBuilder; +use tlsfp_core::ja4::{Transport, ja4}; +use tlsfp_core::parse::parse_client_hello; + +const FOXIO_HASH: &str = "t13d1516h2_8daaf6152771_e5627efa2ab1"; +const FOXIO_RAW: &str = "t13d1516h2_002f,0035,009c,009d,1301,1302,1303,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0005,000a,000b,000d,0012,0015,0017,001b,0023,002b,002d,0033,4469,ff01_0403,0804,0401,0503,0805,0501,0806,0601"; + +const CIPHERS_ORIGINAL_ORDER: [u16; 15] = [ + 0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030, 0xcca9, 0xcca8, 0xc013, 0xc014, 0x009c, + 0x009d, 0x002f, 0x0035, +]; + +const OPAQUE_EXTENSIONS: [u16; 10] = [ + 0x0005, 0x0012, 0x0015, 0x0017, 0x001b, 0x0023, 0x002d, 0x0033, 0x4469, 0xff01, +]; + +const SIG_ALGS: [u16; 8] = [ + 0x0403, 0x0804, 0x0401, 0x0503, 0x0805, 0x0501, 0x0806, 0x0601, +]; + +/// Assembles the handshake body for the canonical FoxIO example. +/// +/// The cipher suites are supplied out of order with an injected GREASE value, to +/// prove that sorting and GREASE removal happen before hashing. When `reversed` +/// is set the opaque extensions are added in reverse, which must not change the +/// fingerprint because JA4 sorts the extension list. +fn foxio_body(reversed: bool) -> Vec { + let mut ciphers = vec![0x0a0a]; + ciphers.extend_from_slice(&CIPHERS_ORIGINAL_ORDER); + + let mut builder = ClientHelloBuilder::new() + .ciphers(&ciphers) + .sni("example.com") + .supported_groups(&[0x001d, 0x0017]) + .ec_point_formats(&[0x00]) + .signature_algorithms(&SIG_ALGS) + .alpn(&[b"h2"]) + .supported_versions(&[0x2a2a, 0x0304, 0x0303]); + + if reversed { + for ext in OPAQUE_EXTENSIONS.iter().rev() { + builder = builder.extension(*ext, vec![]); + } + } else { + for ext in OPAQUE_EXTENSIONS { + builder = builder.extension(ext, vec![]); + } + } + + builder.build_body() +} + +#[test] +fn reproduces_foxio_canonical_example() { + let body = foxio_body(false); + let ch = parse_client_hello(&body).unwrap(); + let fp = ja4(&ch, Transport::Tcp); + assert_eq!(fp.raw, FOXIO_RAW); + assert_eq!(fp.hash, FOXIO_HASH); +} + +#[test] +fn extension_order_does_not_change_the_hash() { + let normal = foxio_body(false); + let reversed = foxio_body(true); + let normal = parse_client_hello(&normal).unwrap(); + let reversed = parse_client_hello(&reversed).unwrap(); + assert_eq!( + ja4(&normal, Transport::Tcp).hash, + ja4(&reversed, Transport::Tcp).hash + ); +} + +#[test] +fn transport_marker_switches_first_character() { + let body = foxio_body(false); + let ch = parse_client_hello(&body).unwrap(); + let tcp = ja4(&ch, Transport::Tcp); + let quic = ja4(&ch, Transport::Quic); + assert!(tcp.hash.starts_with('t')); + assert!(quic.hash.starts_with('q')); + assert_eq!(&tcp.hash[1..], &quic.hash[1..]); +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/ja4_suite.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/ja4_suite.rs new file mode 100644 index 00000000..be34d979 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/ja4_suite.rs @@ -0,0 +1,64 @@ +// ©AngelaMos | 2026 +// ja4_suite.rs + +use tlsfp_core::ja4::{Transport, ja4s}; +use tlsfp_core::ja4h::{ja4h, parse_http_request}; +use tlsfp_core::parse::parse_server_hello; + +/// Builds the ServerHello behind the FoxIO JA4S example: TLS 1.3 selected via the +/// supported versions extension, the cipher 0x1301, and two extensions in the +/// order key share then supported versions. +fn foxio_server_hello() -> Vec { + let mut body = Vec::new(); + body.extend_from_slice(&0x0303u16.to_be_bytes()); + body.extend_from_slice(&[0u8; 32]); + body.push(0); + body.extend_from_slice(&0x1301u16.to_be_bytes()); + body.push(0); + + let mut exts = Vec::new(); + exts.extend_from_slice(&0x0033u16.to_be_bytes()); + exts.extend_from_slice(&0u16.to_be_bytes()); + exts.extend_from_slice(&0x002bu16.to_be_bytes()); + exts.extend_from_slice(&2u16.to_be_bytes()); + exts.extend_from_slice(&0x0304u16.to_be_bytes()); + + body.extend_from_slice(&u16::try_from(exts.len()).unwrap().to_be_bytes()); + body.extend_from_slice(&exts); + body +} + +#[test] +fn ja4s_reproduces_foxio_example() { + let body = foxio_server_hello(); + let sh = parse_server_hello(&body).unwrap(); + let fp = ja4s(&sh, Transport::Tcp); + assert_eq!(fp.hash, "t130200_1301_234ea6891581"); + assert_eq!(fp.raw, "t130200_1301_0033,002b"); +} + +/// The published JA4H example for a request with four uppercase headers, no +/// cookies, no referer, and no accept language. The header hash is the SHA256 of +/// the comma joined header names in wire order. +#[test] +fn ja4h_reproduces_published_example() { + let raw = b"GET * HTTP/1.1\r\nHOST: a\r\nMAN: b\r\nMX: c\r\nST: d\r\n\r\n"; + let req = parse_http_request(raw).unwrap(); + let fp = ja4h(&req); + assert_eq!( + fp.hash, + "ge11nn040000_a3c882e23515_000000000000_000000000000" + ); +} + +/// Cookies and a referer must flip their flags and stop being counted as plain +/// headers, while the accept language must populate the language characters. +#[test] +fn ja4h_flags_cookies_referer_and_language() { + let raw = b"GET / HTTP/1.1\r\nHost: x\r\nAccept-Language: en-US,en;q=0.9\r\nReferer: http://x\r\nCookie: a=1; b=2\r\n\r\n"; + let req = parse_http_request(raw).unwrap(); + let fp = ja4h(&req); + let prefix = fp.hash.split('_').next().unwrap(); + assert_eq!(prefix, "ge11cr02enus"); + assert!(!fp.hash.contains("_000000000000_000000000000")); +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/parse.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/parse.rs new file mode 100644 index 00000000..5d1dcf3c --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/parse.rs @@ -0,0 +1,105 @@ +// ©AngelaMos | 2026 +// parse.rs + +mod common; + +use common::ClientHelloBuilder; +use proptest::prelude::*; +use tlsfp_core::parse::{ + first_handshake_message, handshake_bytes, parse_client_hello, parse_server_hello, +}; +use tlsfp_core::registry::handshake_type; + +fn sample() -> ClientHelloBuilder { + ClientHelloBuilder::new() + .ciphers(&[0x0a0a, 0x1301, 0x1302, 0x1303, 0xc02b]) + .sni("example.com") + .supported_groups(&[0x1a1a, 0x001d, 0x0017]) + .ec_point_formats(&[0x00]) + .signature_algorithms(&[0x0403, 0x0804, 0x0401]) + .alpn(&[b"h2", b"http/1.1"]) + .supported_versions(&[0x2a2a, 0x0304, 0x0303]) +} + +#[test] +fn parses_every_field_a_fingerprint_reads() { + let body = sample().build_body(); + let ch = parse_client_hello(&body).unwrap(); + + assert_eq!(ch.legacy_version, 0x0303); + assert_eq!( + ch.cipher_suites.as_slice(), + &[0x0a0a, 0x1301, 0x1302, 0x1303, 0xc02b] + ); + assert_eq!(ch.extensions.len(), 6); + assert_eq!(ch.server_name(), Some("example.com")); + assert_eq!(ch.supported_groups().as_slice(), &[0x1a1a, 0x001d, 0x0017]); + assert_eq!(ch.ec_point_formats().as_slice(), &[0x00]); + assert_eq!( + ch.signature_algorithms().as_slice(), + &[0x0403, 0x0804, 0x0401] + ); + assert_eq!(ch.alpn_protocols().first().copied(), Some(b"h2".as_slice())); + assert_eq!( + ch.supported_versions().as_slice(), + &[0x2a2a, 0x0304, 0x0303] + ); + assert!(!ch.is_sslv2); +} + +#[test] +fn reads_client_hello_through_the_record_layer() { + let stream = sample().build_record(); + let hs = handshake_bytes(&stream).unwrap(); + let body = first_handshake_message(&hs, handshake_type::CLIENT_HELLO).unwrap(); + let ch = parse_client_hello(body).unwrap(); + assert_eq!(ch.cipher_suites.len(), 5); +} + +#[test] +fn server_hello_parses_single_cipher() { + let mut body = Vec::new(); + body.extend_from_slice(&0x0303u16.to_be_bytes()); + body.extend_from_slice(&[0u8; 32]); + body.push(0); + body.extend_from_slice(&0x1301u16.to_be_bytes()); + body.push(0); + body.extend_from_slice(&0u16.to_be_bytes()); + + let sh = parse_server_hello(&body).unwrap(); + assert_eq!(sh.cipher_suite, 0x1301); + assert_eq!(sh.legacy_version, 0x0303); +} + +#[test] +fn client_hello_without_extensions_parses() { + let body = ClientHelloBuilder::new() + .ciphers(&[0x002f, 0x0035]) + .build_body(); + let body = &body[..body.len() - 2]; + let ch = parse_client_hello(body).unwrap(); + assert_eq!(ch.cipher_suites.as_slice(), &[0x002f, 0x0035]); + assert!(ch.extensions.is_empty()); +} + +proptest! { + #[test] + fn parser_never_panics_on_arbitrary_bytes(bytes in proptest::collection::vec(any::(), 0..2048)) { + let _ = parse_client_hello(&bytes); + let _ = parse_server_hello(&bytes); + if let Ok(hs) = handshake_bytes(&bytes) { + let _ = first_handshake_message(&hs, handshake_type::CLIENT_HELLO); + } + } + + #[test] + fn well_formed_prefix_with_random_extension_tail_is_stable( + tail in proptest::collection::vec(any::(), 0..256) + ) { + let mut ch = ClientHelloBuilder::new().ciphers(&[0x1301, 0x1302]); + ch = ch.extension(0xabcd, tail); + let body = ch.build_body(); + let parsed = parse_client_hello(&body).unwrap(); + prop_assert_eq!(parsed.cipher_suites.as_slice(), &[0x1301, 0x1302]); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/pipeline.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/pipeline.rs new file mode 100644 index 00000000..3c670e94 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/pipeline.rs @@ -0,0 +1,196 @@ +// ©AngelaMos | 2026 +// pipeline.rs + +//! Full pipeline known answer tests over the vendored FoxIO captures. +//! +//! These read a real capture file off disk, run it through the entire stack +//! the binary uses, and assert the fingerprints against the values FoxIO +//! published for the same files. A unit test proves an algorithm in isolation; +//! these prove that the file reader, the link layer decoder, the TCP +//! reassembler, the protocol sniffer, and the algorithms together still land +//! on the published answer. Every hash asserted here was independently +//! recomputed from its raw pre hash string with a shell tool before being +//! pinned, so a regression in any stage shows up as a mismatch rather than a +//! silently wrong but self consistent number. + +use std::path::PathBuf; + +use tlsfp_core::pipeline::event::StreamEvent; +use tlsfp_core::{FingerprintEvent, PcapFileSource, Pipeline, PipelineConfig}; + +fn pcap_path(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../testdata/pcap") + .join(name) +} + +/// Runs one capture all the way through the pipeline and returns its events. +fn fingerprint(name: &str) -> Vec { + let mut source = + PcapFileSource::open(pcap_path(name)).unwrap_or_else(|e| panic!("opening {name}: {e}")); + let mut pipeline = Pipeline::new(PipelineConfig::default()); + let mut events = Vec::new(); + pipeline + .run(&mut source, |event| events.push(event)) + .unwrap_or_else(|e| panic!("running {name}: {e}")); + events +} + +fn client_hellos(events: &[FingerprintEvent]) -> Vec<(&str, &str)> { + events + .iter() + .filter_map(|e| match &e.event { + StreamEvent::ClientHello { ja4, .. } => Some((ja4.hash.as_str(), ja4.raw.as_str())), + _ => None, + }) + .collect() +} + +fn ja4s_hashes(events: &[FingerprintEvent]) -> Vec<(&str, &str)> { + events + .iter() + .filter_map(|e| match &e.event { + StreamEvent::ServerHello { ja4s, .. } => Some((ja4s.hash.as_str(), ja4s.raw.as_str())), + _ => None, + }) + .collect() +} + +fn ja4x_hashes(events: &[FingerprintEvent]) -> Vec<&str> { + events + .iter() + .filter_map(|e| match &e.event { + StreamEvent::Certificate { ja4x } => Some(ja4x.as_str()), + _ => None, + }) + .collect() +} + +fn ja4h_hashes(events: &[FingerprintEvent]) -> Vec<&str> { + events + .iter() + .filter_map(|e| match &e.event { + StreamEvent::HttpRequest { ja4h, .. } => Some(ja4h.hash.as_str()), + _ => None, + }) + .collect() +} + +#[test] +fn tls_alpn_h2_reproduces_published_ja4_and_ja4x() { + let events = fingerprint("tls-alpn-h2.pcap"); + + let hellos = client_hellos(&events); + assert_eq!(hellos.len(), 1); + assert_eq!(hellos[0].0, "t12d4605h2_85626a9a5f7f_aaf95bb78ec9"); + + let certs = ja4x_hashes(&events); + assert_eq!( + certs.first().copied(), + Some("7d5dbb3783b4_ba7ce0880c07_7bf9a7bf7029") + ); +} + +#[test] +fn chrome_cloudflare_tcp_handshake_reproduces_published_ja4() { + let events = fingerprint("chrome-cloudflare-quic-with-secrets.pcapng"); + + let hellos = client_hellos(&events); + assert!( + hellos + .iter() + .any(|(hash, _)| *hash == "t13d1516h2_8daaf6152771_e5627efa2ab1"), + "expected the published chrome JA4, saw {hellos:?}" + ); + + let servers = ja4s_hashes(&events); + assert!( + servers + .iter() + .any(|(hash, _)| *hash == "t130200_1301_234ea6891581"), + "expected the published cloudflare JA4S, saw {servers:?}" + ); +} + +#[test] +fn tls_handshake_reproduces_published_ja4s_raw_and_hash() { + let events = fingerprint("tls-handshake.pcapng"); + + // The capture holds many flows to many Google and Cloudflare endpoints, so + // several distinct JA4S values appear. The published vector is the one for + // the first flow, and it must carry both its published hash and raw form. + let servers = ja4s_hashes(&events); + assert!( + servers + .iter() + .any(|(hash, raw)| *hash == "t130200_1301_234ea6891581" + && *raw == "t130200_1301_0033,002b"), + "expected the published JA4S with its raw form, saw {servers:?}" + ); + + let hellos = client_hellos(&events); + assert!(hellos.len() >= 5); + assert!( + hellos + .iter() + .any(|(hash, _)| *hash == "t13d1516h2_8daaf6152771_e5627efa2ab1") + ); +} + +#[test] +fn http1_with_cookies_reproduces_pinned_ja4h() { + let events = fingerprint("http1-with-cookies.pcapng"); + let hashes = ja4h_hashes(&events); + assert_eq!( + hashes, + vec!["ge11cr04da00_8ddaef5d77af_280f366eaa04_c2fb0fe53442"] + ); +} + +#[test] +fn tls12_capture_recomputes_to_pinned_ja4() { + let events = fingerprint("tls12.pcap"); + let hellos = client_hellos(&events); + assert_eq!(hellos.len(), 1); + assert_eq!(hellos[0].0, "t13d1715h2_5b57614c22b0_3d5424432f57"); +} + +#[test] +fn non_ascii_alpn_follows_the_spec_hex_rule() { + let events = fingerprint("tls-non-ascii-alpn.pcapng"); + let hellos = client_hellos(&events); + assert_eq!(hellos.len(), 1); + let (hash, _) = hellos[0]; + let alpn_chars = &hash[8..10]; + assert_eq!( + alpn_chars, "bd", + "spec rule prints first and last hex chars of the raw ALPN value" + ); +} + +#[test] +fn browsers_x509_extracts_every_certificate_in_chain_order() { + let events = fingerprint("browsers-x509.pcapng"); + let certs = ja4x_hashes(&events); + assert!( + certs.len() >= 7, + "expected several certificates, saw {}", + certs.len() + ); + assert!(certs.iter().all(|c| c.matches('_').count() == 2)); +} + +#[test] +fn tunneled_capture_yields_nothing_without_crashing() { + let events = fingerprint("gre-erspan-vxlan.pcap"); + assert!(events.is_empty()); +} + +#[test] +fn evasion_capture_is_processed_without_panicking() { + let events = fingerprint("CVE-2018-6794.pcap"); + assert!( + ja4h_hashes(&events).len() >= 2, + "the reassembler should still recover the HTTP requests" + ); +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/reassembly.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/reassembly.rs new file mode 100644 index 00000000..31a08ac4 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/reassembly.rs @@ -0,0 +1,114 @@ +// ©AngelaMos | 2026 +// reassembly.rs + +//! Property tests for the TCP stream reassembler. +//! +//! The reassembler is the component an adversary reaches first: it consumes +//! attacker controlled sequence numbers and segment boundaries on every +//! connection. Two properties pin it down. The first is correctness under +//! reordering: any stream cut into any segments and delivered in any order +//! must reassemble back to the original bytes, because that is the whole job. +//! The second is the absence of panics under fully arbitrary input, because a +//! passive sensor that can be crashed by a crafted segment is a denial of +//! service waiting to happen. + +use proptest::prelude::*; +use tlsfp_core::pipeline::flow::{PushOutcome, ReassemblyLimits, StreamReassembler}; + +fn generous_limits() -> ReassemblyLimits { + ReassemblyLimits { + max_assembled_bytes: 1 << 20, + max_pending_bytes: 1 << 20, + max_pending_segments: 4096, + } +} + +/// A stream plus its segment boundaries and a delivery permutation. +fn stream_and_delivery() -> impl Strategy, Vec<(usize, usize)>, Vec)> { + prop::collection::vec(any::(), 1..2048).prop_flat_map(|data| { + let len = data.len(); + let cuts = prop::collection::vec(0..len, 0..16); + (Just(data), cuts).prop_flat_map(|(data, mut cuts)| { + cuts.sort_unstable(); + cuts.dedup(); + let boundaries = boundaries_from_cuts(&cuts, data.len()); + let permutation = Just((0..boundaries.len()).collect::>()).prop_shuffle(); + (Just(data), Just(boundaries), permutation) + }) + }) +} + +fn boundaries_from_cuts(cuts: &[usize], len: usize) -> Vec<(usize, usize)> { + let mut points = vec![0]; + points.extend(cuts.iter().copied().filter(|&c| c > 0 && c < len)); + points.push(len); + points.dedup(); + points.windows(2).map(|w| (w[0], w[1])).collect() +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + + /// Any segmentation delivered in any order reassembles to the original. + #[test] + fn arbitrary_segmentation_and_reordering_reconstructs_the_stream( + (data, boundaries, permutation) in stream_and_delivery(), + ) { + let base_seq = 0x1234_5678u32; + let mut reasm = StreamReassembler::new(generous_limits()); + reasm.anchor(base_seq); + + for &segment_index in &permutation { + let (start, end) = boundaries[segment_index]; + let seq = base_seq.wrapping_add(u32::try_from(start).unwrap()); + reasm.push(seq, &data[start..end]); + } + + prop_assert_eq!(reasm.data(), data.as_slice()); + } + + /// Duplicates layered on top of a complete stream change nothing. + #[test] + fn duplicate_delivery_is_idempotent( + (data, boundaries, permutation) in stream_and_delivery(), + ) { + let base_seq = 42u32; + let mut reasm = StreamReassembler::new(generous_limits()); + reasm.anchor(base_seq); + + for round in 0..2 { + for &segment_index in &permutation { + let (start, end) = boundaries[segment_index]; + let seq = base_seq.wrapping_add(u32::try_from(start).unwrap()); + let outcome = reasm.push(seq, &data[start..end]); + if round == 1 { + prop_assert!(matches!( + outcome, + PushOutcome::Unchanged | PushOutcome::Grew + )); + } + } + } + prop_assert_eq!(reasm.data(), data.as_slice()); + } + + /// Fully arbitrary segments never panic and never exceed the cap. + #[test] + fn arbitrary_input_never_panics_or_overruns( + segments in prop::collection::vec( + (any::(), prop::collection::vec(any::(), 0..64)), + 0..256, + ), + ) { + let limits = ReassemblyLimits { + max_assembled_bytes: 512, + max_pending_bytes: 512, + max_pending_segments: 32, + }; + let mut reasm = StreamReassembler::new(limits); + for (seq, payload) in &segments { + reasm.push(*seq, payload); + prop_assert!(reasm.data().len() <= limits.max_assembled_bytes); + } + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/Cargo.toml b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/Cargo.toml new file mode 100644 index 00000000..80fbf70b --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/Cargo.toml @@ -0,0 +1,38 @@ +# ©AngelaMos | 2026 +# Cargo.toml + +[package] +name = "tlsfp" +description = "JA3/JA4 TLS fingerprinting tool with live capture, intel matching, and anomaly detection" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +default-run = "tlsfp" + +[[bin]] +name = "tlsfp" +path = "src/main.rs" + +[lints] +workspace = true + +[dependencies] +tlsfp-core.workspace = true +anyhow.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +pcap-parser.workspace = true +pcap.workspace = true +tokio.workspace = true +flume.workspace = true +rusqlite.workspace = true +axum.workspace = true +tower-http.workspace = true +tower.workspace = true +serde.workspace = true +serde_json.workspace = true +clap.workspace = true +etherparse.workspace = true diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/cli.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/cli.rs new file mode 100644 index 00000000..47b061ba --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/cli.rs @@ -0,0 +1,126 @@ +// ©AngelaMos | 2026 +// cli.rs + +use anyhow::{Context, Result}; +use clap::{Parser, Subcommand}; +use tracing_subscriber::EnvFilter; + +use tlsfp_core::{PcapFileSource, Pipeline, PipelineConfig}; + +/// JA3/JA4 TLS fingerprinting tool. +/// +/// Fingerprints TLS clients and servers from live capture or packet captures, +/// matches them against a local intelligence database, and flags anomalies such +/// as a fingerprint that disagrees with its own User-Agent. +#[derive(Debug, Parser)] +#[command(name = "tlsfp", version, about, long_about = None)] +pub struct Cli { + /// Increase log verbosity (repeat for more detail). + #[arg(short, long, global = true, action = clap::ArgAction::Count)] + pub verbose: u8, + + #[command(subcommand)] + pub command: Command, +} + +#[derive(Debug, Subcommand)] +pub enum Command { + /// Fingerprint every TLS and QUIC handshake in a packet capture file. + Pcap { + /// Path to a pcap or pcapng file. + path: std::path::PathBuf, + + /// Emit one JSON object per event instead of readable lines. + #[arg(long)] + json: bool, + }, + + /// Capture live from a network interface and fingerprint in real time. + Live { + /// Interface name, for example eth0. + interface: String, + }, + + /// Serve the web dashboard and HTTP API. + Serve { + /// Address to bind, for example 127.0.0.1:8080. + #[arg(default_value = "127.0.0.1:8080")] + bind: String, + }, +} + +impl Cli { + pub fn init_tracing(&self) { + let default = match self.verbose { + 0 => "tlsfp=info", + 1 => "tlsfp=debug", + _ => "tlsfp=trace", + }; + let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(default)); + tracing_subscriber::fmt().with_env_filter(filter).init(); + } + + pub fn run(self) -> Result<()> { + match self.command { + Command::Pcap { path, json } => run_pcap(&path, json), + Command::Live { interface } => { + anyhow::bail!("live capture on {interface} is not wired up yet") + } + Command::Serve { bind } => { + anyhow::bail!("dashboard on {bind} is not wired up yet") + } + } + } +} + +/// Fingerprints a capture file and prints one event per line on stdout. +/// +/// The summary goes to the log rather than stdout so that piping the output +/// into a tool sees only events, while a human still learns how much of the +/// capture was readable and whether the file was cut short mid packet. +fn run_pcap(path: &std::path::Path, json: bool) -> Result<()> { + let mut source = PcapFileSource::open(path) + .with_context(|| format!("cannot open capture {}", path.display()))?; + let mut pipeline = Pipeline::new(PipelineConfig::default()); + + let stdout = std::io::stdout().lock(); + let mut out = std::io::BufWriter::new(stdout); + let mut write_failure = None; + pipeline.run(&mut source, |event| { + use std::io::Write as _; + let result = if json { + serde_json::to_writer(&mut out, &event) + .map_err(anyhow::Error::from) + .and_then(|()| writeln!(out).map_err(anyhow::Error::from)) + } else { + writeln!(out, "{event}").map_err(anyhow::Error::from) + }; + if write_failure.is_none() { + if let Err(error) = result { + write_failure = Some(error); + } + } + })?; + if let Some(error) = write_failure { + return Err(error.context("writing events to stdout")); + } + { + use std::io::Write as _; + out.flush().context("flushing events to stdout")?; + } + + let counters = pipeline.counters(); + tracing::info!( + frames = counters.frames, + tcp_segments = counters.tcp_segments, + events = counters.events, + flows = counters.flows_created, + unfinished_tls_streams = counters.unfinished_tls_streams, + segments_dropped = counters.segments_dropped, + "capture processed" + ); + if source.truncated() { + tracing::warn!("capture file ended mid packet; the tail was not read"); + } + Ok(()) +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/main.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/main.rs new file mode 100644 index 00000000..ce70db7b --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/main.rs @@ -0,0 +1,15 @@ +// ©AngelaMos | 2026 +// main.rs + +mod cli; + +use anyhow::Result; +use clap::Parser; + +use crate::cli::Cli; + +fn main() -> Result<()> { + let cli = Cli::parse(); + cli.init_tracing(); + cli.run() +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/deny.toml b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/deny.toml new file mode 100644 index 00000000..2c46c991 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/deny.toml @@ -0,0 +1,30 @@ +# ©AngelaMos | 2026 +# deny.toml + +[graph] +targets = ["x86_64-unknown-linux-gnu"] +all-features = true + +[advisories] +version = 2 +yanked = "deny" + +[licenses] +version = 2 +confidence-threshold = 0.93 +allow = [ + "MIT", + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "Zlib", + "Unicode-3.0", + "Unicode-DFS-2016", + "MPL-2.0", + "CDLA-Permissive-2.0", +] + +[bans] +multiple-versions = "warn" +wildcards = "deny" diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/rustfmt.toml b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/rustfmt.toml new file mode 100644 index 00000000..c59829cf --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/rustfmt.toml @@ -0,0 +1,4 @@ +# ©AngelaMos | 2026 +# rustfmt.toml + +edition = "2024" diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/CVE-2018-6794.pcap b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/CVE-2018-6794.pcap new file mode 100644 index 00000000..3ce5450f Binary files /dev/null and b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/CVE-2018-6794.pcap differ diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/PROVENANCE.md b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/PROVENANCE.md new file mode 100644 index 00000000..f0ecffe0 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/PROVENANCE.md @@ -0,0 +1,49 @@ + + + +# Test capture provenance + +Every capture file in this directory is vendored, bit for bit, from the +FoxIO JA4 reference repository. None of them were created or modified here, +and the SHA-256 digests below let anyone verify that claim offline. + +## Source + +- Repository: +- Commit: `4ab8e3f18f4b27e0b896c4ec2c251e20506eb87e` (fetched 2026-06-10) +- Path within the repository: `pcap/` +- Fetch URL pattern: `https://raw.githubusercontent.com/FoxIO-LLC/ja4//pcap/` + +## Licensing + +The FoxIO repository carries two licenses at the commit above: `LICENSE-JA4` +(BSD 3-Clause, covering the JA4 TLS client fingerprint) and `LICENSE` +(FoxIO License 1.1, covering the JA4+ suite). These capture files are the +repository's own test fixtures and are redistributed here unmodified, with +attribution, solely as test inputs for this non-commercial educational +project. See `NOTICE.md` at the project root for how the licensing split +applies to this project as a whole. + +## Files + +| File | SHA-256 | Why it is here | +| --- | --- | --- | +| `tls-handshake.pcapng` | `5a0c9f3d0f437e16fc68c3ce0d87998edf4f229b335c391361ed301bd22a513e` | TLS 1.3 handshake; anchors the published JA4S vector `t130200_1301_234ea6891581` | +| `tls-alpn-h2.pcap` | `8c00fd3e6c370b39dac61ad3a15c693088f74f3dbc836ee4e8f57105b1e84a91` | TLS 1.2 with ALPN h2; anchors the published JA4 vector `t12d4605h2_85626a9a5f7f_aaf95bb78ec9` and the JA4X DigiCert chain vectors | +| `tls12.pcap` | `d8c9ae8781c9bbba3a1bf5a95d7a6f309a3edd14c64a7c8adbc673d337fd5af4` | Minimal TLS 1.2 ClientHello | +| `tls-non-ascii-alpn.pcapng` | `cf1dd939619b8d65904dfd23b4f21c3255b6f513dc2e12117d5de2633d063f71` | ALPN value with non-ASCII bytes; exercises the spec-vs-reference divergence (FoxIO issue 178) | +| `chrome-cloudflare-quic-with-secrets.pcapng` | `b7c9de1238aef44d53dbe1add125a7b9e344e9063b98850c78dec23632b83942` | Chrome to Cloudflare; TCP stream anchors the published JA4 vector `t13d1516h2_8daaf6152771_e5627efa2ab1`, QUIC stream reserved for the QUIC milestone | +| `browsers-x509.pcapng` | `e05937fe5f3659f1b94b46305e419b878ba309ebcba8308750acaac704112906` | Browser certificate chains for JA4X over real captures | +| `http1-with-cookies.pcapng` | `7083ca41bcd09b21cb92e7f2d5bd09d73f25703be8555bb82642c2495eb15ef9` | Cleartext HTTP/1.1 request with cookies for JA4H over a reassembled stream | +| `gre-erspan-vxlan.pcap` | `5bbdb30a0707e21070ece3cd26068f72c34a0750d97c1e7720166cb4d0baf6d6` | Tunneled traffic; proves the decoder skips what it does not understand instead of crashing | +| `CVE-2018-6794.pcap` | `d1aa18b493bc68bf7cb367ce5fc1ee493262b47baff886bbe42533e7495d8b1d` | TCP stream evasion capture; torture input for the reassembler | +| `quic-with-several-tls-frames.pcapng` | `8d2c8a6787b942091aa63e33bde9f28c214b306dc024a66c552077ad30640e71` | QUIC initial with CRYPTO frames split across packets, reserved for the QUIC milestone | + +## Verifying + +```sh +cd testdata/pcap && sha256sum *.pcap *.pcapng +``` + +Compare the output against the table above. Any mismatch means a file no +longer matches what FoxIO published at the pinned commit. diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/browsers-x509.pcapng b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/browsers-x509.pcapng new file mode 100644 index 00000000..357b734c Binary files /dev/null and b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/browsers-x509.pcapng differ diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/chrome-cloudflare-quic-with-secrets.pcapng b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/chrome-cloudflare-quic-with-secrets.pcapng new file mode 100644 index 00000000..4968365a Binary files /dev/null and b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/chrome-cloudflare-quic-with-secrets.pcapng differ diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/gre-erspan-vxlan.pcap b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/gre-erspan-vxlan.pcap new file mode 100644 index 00000000..ee4d520b Binary files /dev/null and b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/gre-erspan-vxlan.pcap differ diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/http1-with-cookies.pcapng b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/http1-with-cookies.pcapng new file mode 100644 index 00000000..37cfb41f Binary files /dev/null and b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/http1-with-cookies.pcapng differ diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/quic-with-several-tls-frames.pcapng b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/quic-with-several-tls-frames.pcapng new file mode 100644 index 00000000..97e89254 Binary files /dev/null and b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/quic-with-several-tls-frames.pcapng differ diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls-alpn-h2.pcap b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls-alpn-h2.pcap new file mode 100644 index 00000000..74bf35ac Binary files /dev/null and b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls-alpn-h2.pcap differ diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls-handshake.pcapng b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls-handshake.pcapng new file mode 100644 index 00000000..8822715d Binary files /dev/null and b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls-handshake.pcapng differ diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls-non-ascii-alpn.pcapng b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls-non-ascii-alpn.pcapng new file mode 100644 index 00000000..26ce655d Binary files /dev/null and b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls-non-ascii-alpn.pcapng differ diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls12.pcap b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls12.pcap new file mode 100644 index 00000000..c22ee8a1 Binary files /dev/null and b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls12.pcap differ