From 539dd725b9da0e44a0e3d88e741fe8125b5683c0 Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Sun, 8 Feb 2026 18:16:11 -0500 Subject: [PATCH] siem dashbaord learn folder --- .../intermediate/siem-dashboard/README.md | 15 +- .../{docs => assets}/images/alerts.png | Bin .../{docs => assets}/images/dashboard.png | Bin .../{docs => assets}/images/log-viewer.png | Bin .../{docs => assets}/images/rules.png | Bin .../{docs => assets}/images/scenarios.png | Bin .../siem-dashboard/backend/fix_mypy.py | 70 - .../siem-dashboard/backend/wsgi.py | 36 +- .../siem-dashboard/learn/00-OVERVIEW.md | 185 ++ .../siem-dashboard/learn/01-CONCEPTS.md | 321 ++++ .../siem-dashboard/learn/02-ARCHITECTURE.md | 598 ++++++ .../siem-dashboard/learn/03-IMPLEMENTATION.md | 1296 +++++++++++++ .../siem-dashboard/learn/04-CHALLENGES.md | 1597 +++++++++++++++++ 13 files changed, 4038 insertions(+), 80 deletions(-) rename PROJECTS/intermediate/siem-dashboard/{docs => assets}/images/alerts.png (100%) rename PROJECTS/intermediate/siem-dashboard/{docs => assets}/images/dashboard.png (100%) rename PROJECTS/intermediate/siem-dashboard/{docs => assets}/images/log-viewer.png (100%) rename PROJECTS/intermediate/siem-dashboard/{docs => assets}/images/rules.png (100%) rename PROJECTS/intermediate/siem-dashboard/{docs => assets}/images/scenarios.png (100%) delete mode 100644 PROJECTS/intermediate/siem-dashboard/backend/fix_mypy.py create mode 100644 PROJECTS/intermediate/siem-dashboard/learn/00-OVERVIEW.md create mode 100644 PROJECTS/intermediate/siem-dashboard/learn/01-CONCEPTS.md create mode 100644 PROJECTS/intermediate/siem-dashboard/learn/02-ARCHITECTURE.md create mode 100644 PROJECTS/intermediate/siem-dashboard/learn/03-IMPLEMENTATION.md create mode 100644 PROJECTS/intermediate/siem-dashboard/learn/04-CHALLENGES.md diff --git a/PROJECTS/intermediate/siem-dashboard/README.md b/PROJECTS/intermediate/siem-dashboard/README.md index b345eabf..9410c5f1 100644 --- a/PROJECTS/intermediate/siem-dashboard/README.md +++ b/PROJECTS/intermediate/siem-dashboard/README.md @@ -12,19 +12,18 @@ A full-stack Security Information and Event Management dashboard with a built-in Real-time overview of ingested events, active alerts, severity distribution, and top source IPs. -![Dashboard](docs/images/dashboard.png) +![Log Viewer](assets/images/log-viewer.png) ## Log Viewer Paginated, filterable log table showing every event ingested by the system — firewall, auth, IDS, endpoint, DNS, and proxy logs. Click any row to inspect the full normalized payload. -![Log Viewer](docs/images/log-viewer.png) - +![Dashboard](assets/images/dashboard.png) ## Alerts Alerts fire when correlation rules detect suspicious patterns in the event stream. Each alert shows the matched rule, severity, status, grouped source, and the specific events that triggered it. Analysts can acknowledge, investigate, resolve, or mark as false positive. -![Alerts](docs/images/alerts.png) +![Alerts](assets/images/alerts.png) ## Correlation Rules @@ -34,7 +33,7 @@ Define detection logic using three rule types: - **Sequence** — fire when an ordered series of event patterns appears (e.g. failed logins followed by a success) - **Aggregation** — fire when distinct values of a field exceed a threshold (e.g. one IP hitting 10+ ports) -![Rules](docs/images/rules.png) +![Rules](docs/assets/rules.png) ## Scenario Engine @@ -88,9 +87,9 @@ YAML Playbook ──→ Scenario Thread (replays events with timing) ## Quick Start ```bash -git clone -cd siem-dashboard -docker compose -f dev.compose.yml up --build +git clone https://github.com/CarterPerez-dev/Cybersecurity-Projects +cd PROJECTS/intermediate/siem-dashboard +Just dev-up ``` - **App:** http://localhost:8431 diff --git a/PROJECTS/intermediate/siem-dashboard/docs/images/alerts.png b/PROJECTS/intermediate/siem-dashboard/assets/images/alerts.png similarity index 100% rename from PROJECTS/intermediate/siem-dashboard/docs/images/alerts.png rename to PROJECTS/intermediate/siem-dashboard/assets/images/alerts.png diff --git a/PROJECTS/intermediate/siem-dashboard/docs/images/dashboard.png b/PROJECTS/intermediate/siem-dashboard/assets/images/dashboard.png similarity index 100% rename from PROJECTS/intermediate/siem-dashboard/docs/images/dashboard.png rename to PROJECTS/intermediate/siem-dashboard/assets/images/dashboard.png diff --git a/PROJECTS/intermediate/siem-dashboard/docs/images/log-viewer.png b/PROJECTS/intermediate/siem-dashboard/assets/images/log-viewer.png similarity index 100% rename from PROJECTS/intermediate/siem-dashboard/docs/images/log-viewer.png rename to PROJECTS/intermediate/siem-dashboard/assets/images/log-viewer.png diff --git a/PROJECTS/intermediate/siem-dashboard/docs/images/rules.png b/PROJECTS/intermediate/siem-dashboard/assets/images/rules.png similarity index 100% rename from PROJECTS/intermediate/siem-dashboard/docs/images/rules.png rename to PROJECTS/intermediate/siem-dashboard/assets/images/rules.png diff --git a/PROJECTS/intermediate/siem-dashboard/docs/images/scenarios.png b/PROJECTS/intermediate/siem-dashboard/assets/images/scenarios.png similarity index 100% rename from PROJECTS/intermediate/siem-dashboard/docs/images/scenarios.png rename to PROJECTS/intermediate/siem-dashboard/assets/images/scenarios.png diff --git a/PROJECTS/intermediate/siem-dashboard/backend/fix_mypy.py b/PROJECTS/intermediate/siem-dashboard/backend/fix_mypy.py deleted file mode 100644 index a0ce60d2..00000000 --- a/PROJECTS/intermediate/siem-dashboard/backend/fix_mypy.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -©AngelaMos | 2026 -fix_mypy.py -""" - -import re -from pathlib import Path - -def remove_unused_ignores(file_path: Path, lines_with_unused: list[int]) -> None: - """Remove unused type: ignore comments from specific lines""" - with open(file_path) as f: - lines = f.readlines() - - for line_num in lines_with_unused: - idx = line_num - 1 - if idx < len(lines): - line = lines[idx] - line = re.sub(r' # type: ignore\[no-untyped-call\]', '', line) - line = re.sub(r' # type: ignore\[no-any-return\]', '', line) - line = re.sub(r' # type: ignore\[attr-defined\]', '', line) - line = re.sub(r' # type: ignore\[no-untyped-call, no-any-return\]', '', line) - line = re.sub(r' # type: ignore', '', line) - lines[idx] = line - - with open(file_path, 'w') as f: - f.writelines(lines) - -# Clean up User.py -remove_unused_ignores( - Path("app/models/User.py"), - [57, 64, 71, 78, 97, 109, 117, 127, 135, 142, 149, 155] -) - -# Clean up ScenarioRun.py -remove_unused_ignores( - Path("app/models/ScenarioRun.py"), - [63, 71, 80, 81, 89, 97, 104, 111, 120, 127] -) - -# Clean up LogEvent.py -remove_unused_ignores( - Path("app/models/LogEvent.py"), - [100, 113, 156, 163, 170, 217, 241, 280] -) - -# Clean up CorrelationRule.py -remove_unused_ignores( - Path("app/models/CorrelationRule.py"), - [58] -) - -# Clean up Alert.py -remove_unused_ignores( - Path("app/models/Alert.py"), - [88, 125, 132] -) - -# Clean up scenario_ctrl.py -remove_unused_ignores( - Path("app/controllers/scenario_ctrl.py"), - [43, 53, 63, 74] -) - -# Clean up rule_ctrl.py -remove_unused_ignores( - Path("app/controllers/rule_ctrl.py"), - [23, 48, 61, 70] -) - -print("Removed unused type: ignore comments") diff --git a/PROJECTS/intermediate/siem-dashboard/backend/wsgi.py b/PROJECTS/intermediate/siem-dashboard/backend/wsgi.py index 92e73db7..564dac29 100644 --- a/PROJECTS/intermediate/siem-dashboard/backend/wsgi.py +++ b/PROJECTS/intermediate/siem-dashboard/backend/wsgi.py @@ -1,9 +1,41 @@ """ ©AngelaMos | 2026 wsgi.py + +⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⠁⣰⣾⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⠿⣿⣿⣿⣿⣁⣭⣝⢿⡋⠽⢯⣝⡻⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣮⠻⣿⣿⣿⣷⡝⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿ +⣿⣿⣿⣿⣿⣿⣿⣿⣿⠏⢠⣾⣿⣿⣿⣿⣿⣿⣿⣿⡿⣫⢔⣫⣥⣾⣿⣿⣿⣏⠙⣿⣧⢻⣿⣿⣶⣭⣓⢬⡻⣿⣿⣿⣿⣿⣿⣿⣿⣷⡙⣿⣿⣿⣿⣌⢿⣿⣿⣿⣿⣿⣿⣿⣿ +⣿⣿⣿⣿⣿⣿⣿⣿⠃⣴⣿⣿⣿⣿⣿⣿⣿⣿⣿⠟⢔⣵⣿⣿⣿⣿⣿⣿⣿⣿⣧⡈⣿⣧⢻⣿⣿⣿⣿⣷⣄⡈⠻⣿⣿⣿⣿⣿⣿⣿⣷⡘⣿⣿⣿⣿⣆⢿⣿⣿⣿⣿⣿⣿⣿ +⣿⣿⣿⣿⣿⣿⡿⠃⣼⣿⣿⣿⣿⣿⣿⣿⣿⣿⠏⢢⣿⣿⣿⣿⣿⣿⣿⢛⣿⣿⣿⣿⢹⣿⡘⣿⣿⣿⣿⣿⣿⣿⣦⡙⢿⣿⣿⣿⣿⣿⣿⣷⡸⣿⣿⣿⣿⡎⢿⣿⣿⣿⣿⣿⣿ +⣿⣿⣿⣿⣿⣿⢇⣼⣿⣿⣿⣿⣿⣿⣿⣿⣿⡏⣰⣿⣿⣿⣿⣿⡿⢻⠏⣾⣿⣿⣿⣿⡇⣿⡇⡹⣿⣿⣿⣿⣿⣿⣿⣿⣌⢿⡟⢻⣿⢈⢧⠙⣧⢹⡟⢿⣿⣿⡼⣿⣿⣿⣿⣿⣿ +⣿⣿⣿⣿⣿⡏⣼⡟⢩⣿⣿⢛⣿⣿⣿⣿⣿⢡⣿⣿⣿⣿⣿⡟⢡⣿⣸⣿⣿⣿⣿⣿⣷⣿⣇⣳⡿⣿⣿⠻⣿⡉⢻⣿⣿⡎⢿⡴⣝⣋⡌⣎⠸⡏⢿⡌⣿⣿⣧⢹⣿⣿⣿⣿⣿ +⣿⣿⣿⣿⡟⣈⣎⠆⢃⣿⢇⡾⢻⣿⣿⣿⡏⣾⡟⣿⠿⣿⡿⠁⣼⡇⣿⣿⣿⣿⣿⣿⣿⣿⣿⢰⡸⣌⣿⣦⢊⠧⢱⡹⣿⣿⡌⣷⣹⣿⡟⠘⡦⢹⡸⣿⡘⣿⣿⡞⣿⣿⣿⣿⣿ +⣿⣿⣿⡿⡘⣼⠘⡼⣼⡟⡞⣰⣿⣿⣿⣿⠸⣋⢶⡿⢰⣿⢣⣧⣿⢡⣿⢣⢻⢏⣿⣿⣿⣿⡇⣼⡇⡑⣎⠛⣧⡳⠺⣟⠜⣿⣿⡸⣧⢿⣏⣆⢱⡘⡧⣿⣿⣿⣿⡇⢻⣿⣿⣿⣿ +⣿⣿⣿⢡⢱⣇⣸⢣⣿⢰⣿⣿⣸⣿⣿⡏⣆⣿⢸⡇⢸⠏⡞⢸⣿⢸⣿⣼⢰⣾⣏⣿⡟⡿⠇⣿⣿⢰⠹⡨⡜⢷⡁⢬⣧⣸⣿⣇⢻⣼⣿⣿⣷⣿⣧⢸⣿⣿⣿⣿⢸⣿⣿⣿⣿ +⣿⣿⡟⣾⣿⣿⡟⣸⣿⢋⣿⣿⣿⣿⣿⢡⡏⡼⢸⡷⣿⠘⢸⢸⣿⣬⣿⣿⢘⣥⢻⡟⠇⣧⢳⣿⣿⠈⣷⠳⣿⣎⢿⣿⣿⣿⣿⣿⡜⣿⣿⣿⣿⣿⣷⡜⣿⣿⣿⣿⡎⣿⣿⣿⣿ +⣿⣿⢡⣿⣿⣿⡇⣿⣿⣼⣿⣿⣿⣿⣿⢸⣿⡇⣿⣷⠇⠀⡌⢸⣿⢻⣿⣿⢸⣿⡼⢡⢸⡟⣸⣿⣿⡎⣌⢷⡹⣿⣦⠻⣿⣿⣿⣿⣧⢿⣿⣿⣿⣿⣿⡇⣿⣿⣿⣿⡇⣿⣿⣿⣿ +⣿⣿⣾⣿⣿⣿⢷⣿⣿⣿⣿⣿⣿⣿⡿⣿⣿⠇⣿⣿⠀⣼⣇⢸⣿⢸⣿⣿⢸⣿⣷⡏⣿⢱⣿⢻⣿⡇⣎⢣⡱⡜⢿⣷⡈⠻⣿⣿⣿⢸⣿⣿⣿⣿⣿⣧⣿⣿⣿⣿⣷⣿⣿⣿⣿ +⣿⣟⣿⣿⣿⣏⣼⣿⣿⣿⣿⣿⣿⣿⡇⣿⡏⠀⣿⣿⠀⣿⣿⢸⣿⡆⣿⣿⣸⣿⡟⡼⢃⣿⣿⢸⣿⢱⣿⣷⡕⢌⠢⡻⣿⣮⡪⡙⢿⠀⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿ +⣿⣯⣿⣿⡿⢸⣿⣿⣿⣿⣿⣿⣿⣿⡇⣿⠉⣴⢹⣿⣸⣿⣿⡏⣿⣷⢿⣿⡏⡿⠑⠁⢸⣿⡿⢸⣿⢸⣿⣿⣿⣷⣕⠌⠪⣿⣿⡌⠳⡆⣿⣿⢻⣿⣿⣿⡙⣿⣿⣿⣿⣿⣿⣿⣿ +⣿⣿⣿⣿⠃⢳⣼⣿⣿⣿⣿⣿⣿⣿⡇⡏⢸⣿⢸⣿⣿⣿⣿⣷⣸⣿⠈⣿⠇⡀⢀⣾⢸⣿⡇⠼⠟⣸⠿⠿⠿⢛⣛⣥⣤⡀⠐⠶⣷⡄⣿⡿⣸⣿⣿⣿⣼⠸⣿⣿⣿⣿⣿⣿⣿ +⡏⢸⣿⣿⠀⠰⣾⣿⣿⣿⡇⣿⣿⣿⡇⠃⣾⣿⡆⢍⢛⠻⠿⣿⣧⢻⡌⡁⠞⠁⢿⣿⢸⣿⣧⢸⣆⣐⡤⢞⣛⠻⠿⠍⠉⠙⠁⠈⠀⠀⣚⡃⢿⣿⣿⣿⣿⡇⣿⣿⣿⣿⣿⣿⣿ +⣸⢸⣿⡇⠀⡗⣼⣿⣿⣿⣧⢹⣿⣿⣇⠀⣿⣿⣿⡘⠸⡿⢷⣶⣶⣦⠲⢲⡐⢷⡈⡻⣼⡿⣸⠘⣸⣿⣞⠉⢀⡀⡀⠀⠀⠀⢰⣾⣿⠂⢉⢀⣿⣿⣿⣿⣿⢣⣿⣿⣿⣿⣿⣿⣿ +⢧⢸⣿⡇⠃⡇⠟⣿⣿⣿⣿⡞⣿⣿⣿⠀⡿⠇⠉⢁⠀⢀⠀⣀⠀⠀⡀⠀⠈⢎⠳⡄⣿⢡⠇⣰⣿⣿⣧⣾⣿⣇⠑⠮⠔⣁⣾⣿⠛⣠⡾⢸⣿⣿⣿⡏⢣⣿⣿⣿⣿⢿⣿⣿⣿ +⣾⣾⣿⡇⠀⡇⠀⣯⡻⣿⣿⣷⡸⣿⣿⡆⣶⣄⠐⢿⣷⣿⡄⠱⣄⡠⠆⣼⣌⣿⡷⡰⠃⠋⣴⣿⣿⣿⣿⣿⣿⣿⣯⣭⣭⣤⣴⣿⢣⣿⢡⢸⣿⣿⣿⡇⣿⣿⣿⣿⣿⣸⣿⣿⣿ +⣿⡿⣿⡇⡇⡇⢀⢹⣇⣿⣿⣿⣷⡹⣿⣿⡸⣿⣷⣤⠛⣿⠿⠶⠶⢒⣺⣿⣿⡿⢁⣵⣿⣶⣬⣹⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⢇⣾⢃⣸⣼⣿⣿⣿⢠⣿⣿⣿⣿⣿⢿⣿⣿⣿ +⣿⠁⣿⡇⠃⡇⢸⠌⡸⡘⣿⣿⣿⣷⡘⣿⣷⡹⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣴⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡟⡼⣣⡟⡇⣿⣿⣿⢁⣸⣿⣿⣿⣿⣿⢸⣿⣿⣿ +⡛⢰⢻⣧⠀⡇⡸⢀⢣⣷⢻⣿⣿⣿⣿⡌⠻⣷⡽⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⠏⢈⣴⣿⢃⢸⣿⢏⡇⡆⠋⣿⣿⣿⣿⢻⢸⣿⣿⣿ +⣬⠘⡌⣿⢀⡇⡇⢸⡆⡜⣧⢻⣿⣿⣿⣧⢢⣜⢻⣌⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣡⣶⣿⣿⡟⢄⣿⡟⡼⡙⣼⢸⣿⣿⣿⣿⢸⢸⣿⣿⣿ +⣿⡗⡆⠹⡇⢡⡇⢸⡇⡘⣽⣇⢻⣿⣿⣿⡌⢿⡇⢮⡓⠙⠿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣏⡸⠃⣼⣿⣱⠳⢱⣿⢸⣿⣿⣿⣿⢸⡆⣿⣿⣿ +⣿⡇⡄⣷⠹⡼⡇⢸⡇⣷⡙⣿⡄⠻⣿⣿⣿⡌⣿⢸⣷⢕⢦⣬⣛⣿⣿⣿⣿⣿⣿⣿⣿⣿⣟⣛⠿⣿⣿⣿⣿⣿⣿⣿⣿⣿⠟⡅⣼⣿⢡⡇⣰⢸⣿⣸⣿⣿⣿⣿⠈⡇⣿⣿⣿ +⣿⢸⠇⣿⢰⠀⠁⣾⠃⣿⢱⡜⢷⡱⡹⣿⣿⣷⡸⢸⣿⢸⣷⣍⡻⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⢛⢡⡆⣸⡿⣡⢏⣾⣿⢸⣿⣿⣿⣿⣿⣿⠀⣧⣿⣿⣿ +⣿⢸⢰⣿⢸⢸⡆⡙⠀⣿⢸⣗⣌⢧⠱⣜⢿⣿⣷⡘⠟⣼⣿⣿⣿⣷⣮⣝⠻⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⠟⣩⣶⣿⠘⣱⠟⢕⣵⣿⣯⢹⣼⣿⢹⢸⣿⣿⣿⠈⢻⢹⣿⣿ +⡟⣼⢸⣿⠸⣿⢻⡟⡇⣿⢸⠛⣿⣷⡁⠻⣦⡹⣿⣿⣄⢻⣿⢟⣿⣿⣿⣿⠀⢿⡄⣭⣟⡛⠿⣿⣿⣿⠿⣛⣵⣾⣿⣿⠟⡜⢁⡐⣿⣿⣿⣿⡸⡟⣿⢸⢸⣿⣿⣿⠀⡸⡸⣿⣿ +⡇⣿⢸⣿⡇⡏⣼⢡⠇⡿⡾⠀⣿⣿⡿⠆⠈⠵⢎⡻⣿⣦⡁⣾⣿⣿⣿⡏⣼⢸⣧⢻⣿⣿⣿⣶⣶⣶⣾⣿⣿⣿⡿⢋⣨⡆⢿⣇⢻⣿⣿⣿⣇⠃⢻⠘⡈⣿⣿⣿⡇⠃⡇⣿⣿ +⡇⣿⢸⣿⡇⢣⡏⣾⢸⡇⢇⡇⡟⡩⢊⠀⣠⣶⢸⣿⣦⠙⠷⣜⢻⣿⠟⣼⣿⢸⠟⢸⣿⣿⣿⣿⣿⣿⣿⣿⣿⣯⣴⣿⢛⣼⢸⣿⡜⣿⣿⣿⣿⠎⠘⡇⡇⣿⣿⣿⡇⡈⢻⢹⣿ +⡇⣿⢸⣿⡇⡼⢹⠟⢾⣘⣘⡓⡘⠰⢁⣾⣿⣿⢸⡿⣿⢸⣿⣶⠅⣁⣚⣭⣵⣶⠏⡘⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⠟⣡⣿⣿⣇⠻⣧⢹⣿⠏⢱⣿⣆⢧⢧⣿⣿⣿⣇⣧⡜⡎⣿ +⣷⢸⠸⡿⢘⣥⣶⣿⣿⣿⣿⡇⠇⢶⣾⢿⣿⣿⣸⡇⣿⢺⢏⡔⣹⣿⣿⣿⣿⣿⣿⣿⣮⡙⢿⣿⣿⣿⣿⣿⠟⣡⣾⣿⣿⣿⣿⣧⣌⠃⢿⣧⢃⢻⣿⣎⢸⢹⣿⣿⣿⢻⠰⢷⢹ +⣿⡦⢅⣴⣿⣿⣿⣿⣿⣿⣿⣿⡸⡌⢿⣇⣿⣿⡟⡇⢿⠠⡿⢱⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣮⣝⣛⣫⣵⣾⣿⣿⣿⣿⣿⣿⣿⣿⣷⡄⢿⣇⢸⣏⠿⡄⠜⣿⣿⣿⢸⣇⠘⡞ """ from app import create_app - - app = create_app() diff --git a/PROJECTS/intermediate/siem-dashboard/learn/00-OVERVIEW.md b/PROJECTS/intermediate/siem-dashboard/learn/00-OVERVIEW.md new file mode 100644 index 00000000..bcaa725e --- /dev/null +++ b/PROJECTS/intermediate/siem-dashboard/learn/00-OVERVIEW.md @@ -0,0 +1,185 @@ +# SIEM Dashboard + +## What This Is + +A full stack Security Information and Event Management (SIEM) platform that ingests log events from multiple source types, normalizes them into a common schema, classifies severity using pattern matching, and runs a real time correlation engine to generate alerts. It includes a React dashboard for monitoring, investigation, and attack scenario playback. + +The backend is Flask with MongoDB for persistence and Redis Streams for real time event delivery. The frontend is React with TypeScript, Zustand for state, and Server-Sent Events for live updates. + +## Why This Matters + +Every organization with more than a handful of servers needs centralized log monitoring. Without it, you're blind to attacks until the damage is done. Commercial SIEMs like Splunk or Microsoft Sentinel cost tens of thousands per year, and most security teams still struggle with alert fatigue, missed correlations, and slow investigation workflows. + +Building a SIEM from scratch teaches you how these systems actually work under the hood. You'll understand why correlation rules matter, how event normalization prevents data silos, and why real time streaming is critical for incident response. + +**Real world scenarios where this applies:** +- A SOC analyst needs to correlate 20 failed SSH logins from one IP with a successful login 30 seconds later. That's a brute force followed by compromise, and the correlation engine in this project detects exactly that pattern. +- An incident responder pivots from a suspicious source IP to find all related events across firewall, auth, and endpoint logs. The pivot API (`/v1/logs/pivot`) enables this workflow. +- A security team wants to test detection rules against historical data before deploying them. The rule test endpoint (`/v1/rules//test`) replays events through a rule without creating real alerts. + +## What You'll Learn + +This project teaches you how SIEM systems process, correlate, and surface security events. By building it yourself, you'll understand: + +**Security Concepts:** +- Log normalization across different source formats (firewall, IDS, auth, endpoint, DNS, proxy). Each source type has different fields and the normalizer in `app/engine/normalizer.py` maps them to a common schema. +- Severity classification using regex pattern matching against known attack indicators like "brute force", "lateral movement", and "data exfiltration". +- Correlation rule evaluation with sliding windows, including threshold counting, ordered sequence detection, and distinct-value aggregation. +- Alert lifecycle management from initial detection through acknowledgment, investigation, resolution, or false positive classification. + +**Technical Skills:** +- Building a Flask application factory with MongoEngine, Redis, and structured error handling +- Implementing JWT authentication with Argon2id password hashing and timing safe verification to prevent user enumeration +- Using Redis Streams with consumer groups for reliable event delivery and Server-Sent Events for real time browser updates +- Writing a threaded correlation engine that evaluates rules against a continuous event stream +- Building a React frontend with Zustand state management, TanStack Query for data fetching, and SSE integration + +**Tools and Techniques:** +- Docker Compose for local development with MongoDB, Redis, Nginx, Flask, and Vite running together +- YAML-based attack scenario playbooks that simulate real MITRE ATT&CK techniques like brute force (T1110.001) and DNS tunneling (T1048.003) +- Pydantic schemas for request validation with automatic error formatting +- MongoDB aggregation pipelines for timeline bucketing, severity breakdowns, and top source analysis + +## Prerequisites + +Before starting, you should understand: + +**Required knowledge:** +- Python basics including classes, decorators, and type hints. You'll see decorators like `@endpoint(roles=ADMIN)` and `@_register(SourceType.FIREWALL)` throughout the codebase. +- Basic understanding of REST APIs and HTTP. The backend exposes around 30 endpoints under `/v1/`. +- Familiarity with JSON and basic database concepts. MongoDB stores documents as JSON-like structures and you'll work with MongoEngine's ORM. + +**Tools you'll need:** +- Docker and Docker Compose, to run the full stack locally +- Python 3.14+ (the project uses modern type hints like `str | None`) +- Node.js 24+ and pnpm for the frontend +- A tool for making HTTP requests (curl, httpie, or Postman) to test the API directly + +**Helpful but not required:** +- Experience with MongoDB queries and aggregation pipelines +- Familiarity with Redis data structures, particularly Streams +- Basic React and TypeScript knowledge for understanding the frontend + +## Quick Start + +Get the project running locally: + +```bash +# Navigate to the project +cd PROJECTS/intermediate/siem-dashboard + +# Start development environment +docker compose -f dev.compose.yml up --build + +# The app will be available at http://localhost:8431 +# Backend API is at http://localhost:8431/api/v1/ +# Direct backend access at http://localhost:5113 +# Direct frontend dev server at http://localhost:3959 +``` + +Create an admin account: + +```bash +docker exec -it siem-backend-dev flask admin create \ + --username admin \ + --email admin@example.com +``` + +Expected output: `Admin account 'admin' created successfully.` + +Test the API: + +```bash +# Register a regular user +curl -X POST http://localhost:8431/api/v1/auth/register \ + -H "Content-Type: application/json" \ + -d '{"username":"analyst1","email":"analyst@test.com","password":"testpass123"}' + +# Ingest a test log event +curl -X POST http://localhost:8431/api/v1/logs/ingest \ + -H "Content-Type: application/json" \ + -d '{"source_type":"auth","event_type":"login_failure","source_ip":"10.0.0.1","username":"root"}' +``` + +## Project Structure + +``` +siem-dashboard/ +├── backend/ +│ ├── app/ +│ │ ├── __init__.py # Flask app factory +│ │ ├── config.py # Pydantic settings (60+ config values) +│ │ ├── extensions.py # MongoDB and Redis initialization +│ │ ├── cli.py # Flask CLI for admin management +│ │ ├── core/ +│ │ │ ├── auth.py # JWT + Argon2id authentication +│ │ │ ├── streaming.py # Redis Streams + SSE generator +│ │ │ ├── errors.py # Error hierarchy and handlers +│ │ │ ├── rate_limiting.py # Flask-Limiter setup +│ │ │ ├── serialization.py # MongoEngine to JSON conversion +│ │ │ └── decorators/ # @endpoint, @S, @R composable decorators +│ │ ├── engine/ +│ │ │ ├── normalizer.py # Multi-format log normalizer +│ │ │ ├── severity.py # Pattern-based severity classifier +│ │ │ └── correlation.py # Sliding window correlation engine +│ │ ├── models/ # MongoEngine documents (User, LogEvent, Alert, etc.) +│ │ ├── routes/ # Flask blueprints for each resource +│ │ ├── controllers/ # Business logic handlers +│ │ ├── schemas/ # Pydantic request/response schemas +│ │ └── scenarios/ +│ │ ├── playbook.py # YAML playbook parser +│ │ ├── runner.py # Threaded scenario executor +│ │ └── playbooks/ # Attack scenario YAML files +│ ├── pyproject.toml # Python dependencies and tool config +│ └── wsgi.py # Gunicorn entry point +├── frontend/ +│ ├── src/ +│ │ ├── api/hooks/ # TanStack Query hooks for each resource +│ │ ├── api/types/ # Zod schemas and TypeScript types +│ │ ├── core/app/ # Shell layout, routing, protected routes +│ │ ├── core/lib/ # Axios client, error handling, query config +│ │ ├── core/stores/ # Zustand stores (auth, stream, UI) +│ │ ├── core/charts/ # visx chart theme and color constants +│ │ ├── routes/ # Page components (lazy loaded) +│ │ └── config.ts # API endpoints, query keys, constants +│ └── package.json +├── conf/ +│ ├── docker/ # Dockerfiles for dev and prod +│ └── nginx/ # Nginx configs for proxying and SSE +├── compose.yml # Production Docker Compose +└── dev.compose.yml # Development Docker Compose +``` + +## Next Steps + +1. **Understand the concepts** - Read [01-CONCEPTS.md](./01-CONCEPTS.md) to learn about log correlation, event normalization, and SIEM architecture principles +2. **Study the architecture** - Read [02-ARCHITECTURE.md](./02-ARCHITECTURE.md) to see how the backend engine, streaming layer, and frontend connect +3. **Walk through the code** - Read [03-IMPLEMENTATION.md](./03-IMPLEMENTATION.md) for a detailed walkthrough of the correlation engine, normalizer, and streaming pipeline +4. **Extend the project** - Read [04-CHALLENGES.md](./04-CHALLENGES.md) for ideas like adding Sigma rule support, building a threat intelligence feed, or implementing SOAR playbooks + +## Common Issues + +**Backend container keeps restarting** +``` +siem-backend-dev | Connection refused: mongodb://mongo:27017/siem +``` +Solution: MongoDB takes 10-30 seconds to initialize. The healthcheck in `dev.compose.yml` has a `start_period: 30s` but sometimes this isn't enough on slower machines. Wait and check `docker compose logs mongo`. + +**SSE streams disconnect immediately** +The Nginx config at `conf/nginx/dev.nginx` has special handling for SSE endpoints. If you see instant disconnects, verify the regex location block `location ~ ^/api(/v1/(logs|alerts)/stream.*)` is matching your request. The SSE endpoints need `proxy_buffering off` and long read timeouts. + +**Redis Stream errors on startup** +``` +BUSYGROUP Consumer Group name already exists +``` +This is normal. The `ensure_consumer_group()` function in `app/core/streaming.py` wraps the creation in `contextlib.suppress(Exception)` because it's idempotent. The consumer group already exists from a previous run. + +**Rate limiting blocks development requests** +Auth endpoints are limited to 10 requests per minute (`RATELIMIT_AUTH` in `config.py`). General endpoints allow 200 per minute. If you're hammering the login endpoint during testing, wait a minute or temporarily increase the limit in your `.env` file. + +## Related Projects + +If you found this interesting, check out: +- **API Rate Limiter** (advanced) - Deep dive into the rate limiting algorithms this project uses via Flask-Limiter, including sliding window and token bucket strategies +- **Network Traffic Analyzer** - Packet-level analysis using Scapy, complementing the higher level log analysis this SIEM performs +- **Docker Security Auditor** - Container security scanning that could feed events into this SIEM's ingestion pipeline diff --git a/PROJECTS/intermediate/siem-dashboard/learn/01-CONCEPTS.md b/PROJECTS/intermediate/siem-dashboard/learn/01-CONCEPTS.md new file mode 100644 index 00000000..6897c88e --- /dev/null +++ b/PROJECTS/intermediate/siem-dashboard/learn/01-CONCEPTS.md @@ -0,0 +1,321 @@ +# Core Security Concepts + +This document explains the security concepts you'll encounter while building this SIEM dashboard. These aren't just definitions. We'll dig into why they matter, how attacks actually work, and how this project implements defenses. + +## Log Normalization + +### What It Is + +Different security tools produce logs in completely different formats. A firewall log has fields like `action`, `protocol`, and `bytes_sent`. An IDS log has `signature_id`, `classification`, and `priority`. An authentication log has `auth_method`, `result`, and `failure_reason`. Raw, these logs are incomparable. + +Log normalization transforms all these different formats into a common schema so you can query, correlate, and analyze them together. In this project, the normalizer (`app/engine/normalizer.py`) takes raw events from seven source types and maps them into a unified `LogEvent` document with consistent fields like `source_ip`, `dest_ip`, `severity`, `event_type`, and a `normalized` dict for source-specific extras. + +### Why It Matters + +Without normalization, a SOC analyst investigating a suspicious IP would need to manually search each tool's interface separately. With normalized data, a single pivot query returns every firewall connection, every auth attempt, every DNS query, and every endpoint process execution from that IP. + +The 2013 Target breach is a textbook case. Their FireEye IDS generated alerts about malware beaconing, but the team couldn't correlate those alerts with the POS system logs showing data exfiltration. The tools existed in silos. A functioning SIEM with proper normalization would have connected those dots. + +### How It Works + +The normalizer uses a registry pattern. Each source type registers a normalizer function via a decorator: + +``` +Source Type Normalizer Function Extracted Fields +───────────── ───────────────────────── ────────────────────────── +firewall _normalize_firewall() action, protocol, bytes_sent/received +ids _normalize_ids() signature_id, signature_name, priority +auth _normalize_auth() auth_method, result, failure_reason +endpoint _normalize_endpoint() process_name, command_line, file_path +dns _normalize_dns() query, query_type, response_code +proxy _normalize_proxy() url, method, status_code, user_agent +generic _normalize_generic() message (fallback) +``` + +The dispatch is straightforward. `normalize()` extracts common fields (IP addresses, ports, timestamps, hostnames), then calls the source-specific normalizer to extract fields unique to that log type. Both get merged into a single document. + +### Common Pitfalls + +**Mistake 1: Assuming all sources have the same fields** +```python +# Bad - will throw KeyError on auth logs +def process_event(event): + bytes = event["bytes_sent"] # Only firewall events have this + +# Good - use .get() with defaults and check source_type +def process_event(event): + if event.get("source_type") == "firewall": + bytes = event.get("bytes_sent", 0) +``` + +**Mistake 2: Losing the raw data** + +The normalizer in this project preserves the original event in the `raw` DictField on `LogEvent`. This is critical for forensics. Normalization is lossy by definition, and during an investigation you'll often need the original log line. + +## Event Correlation + +### What It Is + +Correlation is the process of connecting multiple individual events into a higher level detection. A single failed login is noise. Twenty failed logins from the same IP in 60 seconds is a brute force attack. A brute force followed by a successful login is a compromise. + +This project implements three correlation strategies in `app/engine/correlation.py`: + +**Threshold** rules fire when the count of matching events for a group key exceeds a limit within a time window. Example: more than 10 `login_failure` events from the same `source_ip` in 300 seconds. + +**Sequence** rules fire when a specific ordered set of event types occurs within a window. Example: `login_failure` (count >= 5) followed by `login_success` from the same `source_ip`. + +**Aggregation** rules fire when the number of distinct values of a field exceeds a threshold. Example: one `source_ip` connects to more than 20 distinct `dest_ip` values in 60 seconds (port scan behavior). + +### Why It Matters + +Without correlation, a SIEM is just a log search engine. Correlation is what turns data into detections. The difference between a SIEM that generates useful alerts and one that drowns analysts in noise comes down to the quality of its correlation rules. + +The 2020 SolarWinds attack went undetected for months partly because existing detection tools were looking at individual events in isolation. The attackers used valid credentials, standard protocols, and legitimate tools. Only correlating multiple weak signals across time (unusual service account behavior + new federation trust + anomalous DNS queries) could have surfaced the intrusion earlier. + +### How It Works + +The correlation engine runs as a daemon thread that consumes events from a Redis Stream: + +``` +Redis Stream (siem:logs) + │ + ▼ +┌───────────────────┐ +│ CorrelationEngine │ Daemon thread, reads via XREADGROUP +│ ._run() loop │ +└────────┬──────────┘ + │ + ▼ +┌───────────────────┐ +│ _process_event() │ Iterates all enabled rules +└────────┬──────────┘ + │ + ┌────┴────┬──────────┐ + ▼ ▼ ▼ +Threshold Sequence Aggregation +evaluator evaluator evaluator + │ │ │ + └────┬────┘──────────┘ + │ + Fire? ──Yes──▶ Alert.create_from_rule() + │ + ▼ + Redis Stream (siem:alerts) + │ + ▼ + SSE to browser +``` + +Each evaluator uses a `CorrelationState` object that maintains sliding windows in memory. The state is thread-safe (uses `threading.Lock`) and tracks per-rule, per-group-key windows of events. When evaluating, it prunes expired entries based on `window_seconds` and checks whether the firing condition is met. + +A cooldown mechanism (`CORRELATION_COOLDOWN_SECONDS`, default 300) prevents the same rule from firing repeatedly for the same group key. Without this, a sustained brute force would generate hundreds of identical alerts. + +### Common Attacks This Detects + +1. **Brute Force (T1110.001)** - Threshold rule: count `login_failure` events grouped by `source_ip`, fire when count > 10 in 300 seconds +2. **Lateral Movement (T1021.004)** - Sequence rule: `login_success` from external IP followed by internal SSH connections from the compromised host +3. **Port Scanning (T1046)** - Aggregation rule: count distinct `dest_ip` values per `source_ip`, fire when > 20 unique destinations in 60 seconds +4. **DNS Tunneling (T1048.003)** - Threshold rule on high-frequency TXT queries to a single domain, or aggregation on distinct query subdomains + +### Defense Strategies + +The correlation engine itself is a defense mechanism. But it needs well-tuned rules. This project ships with scenario playbooks that demonstrate attacks and the rules that detect them. The `brute_force_lateral.yml` playbook, for example, simulates 20 SSH login failures followed by a successful login and lateral movement. A threshold rule on `login_failure` grouped by `source_ip` with threshold 10 and window 300 would catch the brute force phase. + +## Severity Classification + +### What It Is + +Not all security events are equally urgent. A firewall allowing an outbound HTTPS connection is informational. A process executing `cat /etc/shadow` is high severity. Severity classification assigns a priority level to each event so analysts focus on what matters. + +This project classifies events into five levels: `critical`, `high`, `medium`, `low`, and `info`. The classifier in `app/engine/severity.py` uses two approaches: event type lookup (fast, for known event types) and regex pattern matching (flexible, for content-based classification). + +### How It Works + +The classifier checks event types first against frozen sets: + +``` +HIGH_SEVERITY_EVENT_TYPES: privilege_escalation, data_exfiltration, + c2_communication, reverse_shell + +MEDIUM_SEVERITY_EVENT_TYPES: login_failure, port_scan, + firewall_deny, ids_alert +``` + +If the event type doesn't match, it falls through to regex matching. The classifier concatenates relevant text fields (event_type, message, normalized fields like signature_name and command_line) into a searchable string, then runs regex patterns from critical down to low. First match wins. + +Critical patterns include things like `privilege.?escalat`, `ransomware`, and `c2.?beacon`. High patterns include `brute.?force`, `lateral.?movement`, and `reverse.?shell`. The regex uses `re.IGNORECASE` for case insensitive matching and `?` for optional separators (so "privilege escalation" and "privilege_escalation" both match). + +### Common Pitfalls + +**Mistake: Classifying everything as high severity** + +This leads to alert fatigue. If everything is critical, nothing is. The classifier in this project defaults to `info` for events that don't match any pattern. This is intentional. An unknown event type with no suspicious keywords is probably routine. Better to miss a low-confidence detection than to bury real alerts in noise. + +## Authentication and Authorization Security + +### What It Is + +The SIEM itself needs to be secured. If an attacker compromises the SIEM, they can suppress alerts, manipulate rules, and cover their tracks. This project implements JWT-based authentication with Argon2id password hashing and role-based access control. + +### How It Works + +The auth system in `app/core/auth.py` uses `pwdlib` with Argon2id, the winner of the 2015 Password Hashing Competition. Argon2id is memory-hard, meaning it's resistant to GPU-based cracking attacks that make bcrypt and SHA-based hashes vulnerable. + +A critical detail: the `verify_password_timing_safe()` function provides constant-time behavior for failed lookups. When a login attempt uses a username that doesn't exist, the system still performs a hash comparison against a dummy hash (`DUMMY_HASH`). This prevents timing attacks that could enumerate valid usernames by measuring response time differences. + +``` +Login attempt with valid username: + 1. Look up user → found + 2. Verify password against stored hash → ~150ms (Argon2id) + 3. Return result + +Login attempt with invalid username: + 1. Look up user → not found + 2. Verify password against DUMMY_HASH → ~150ms (same timing) + 3. Return "Invalid username or password" +``` + +Without this, an attacker could distinguish "user doesn't exist" (fast response) from "wrong password" (slow response, because Argon2id is intentionally expensive). + +### RBAC Model + +Two roles: `analyst` (default) and `admin`. The `@endpoint(roles=ADMIN)` decorator on admin routes checks the JWT claims. Admins can manage users, change roles, and deactivate accounts. The system protects against demoting the last admin (`User.count_admins()` check in `admin_ctrl.py`). + +## Real-Time Event Streaming + +### What It Is + +A SIEM that only shows data on page refresh is nearly useless during an active incident. Analysts need to see events as they happen. This project uses Redis Streams for internal event delivery and Server-Sent Events (SSE) for browser updates. + +### How It Works + +When a log event is ingested or an alert is created, it's published to a Redis Stream via `XADD`. The correlation engine reads from the log stream using `XREADGROUP` with consumer groups (reliable delivery with acknowledgment). The SSE endpoints use `XREAD` to tail the stream and emit events to connected browsers. + +``` +Log Ingestion ──XADD──▶ Redis Stream (siem:logs) + │ + ┌───────────────┼───────────────┐ + │ │ │ + XREADGROUP XREADGROUP XREAD + │ │ │ + Correlation (Future SSE Generator + Engine consumers) ──▶ Browser + │ + Alert Created ──XADD──▶ Redis Stream (siem:alerts) + │ + XREAD + │ + SSE Generator + ──▶ Browser +``` + +The SSE generator in `app/core/streaming.py` sends keepalive comments (`: keepalive\n\n`) when no events arrive within the block timeout. This prevents proxy timeouts and lets the browser detect dropped connections. The Nginx config disables buffering and sets `proxy_read_timeout 3600s` for SSE endpoints. + +## How These Concepts Relate + +``` +Raw Log Event + │ + ▼ +Normalization (common schema) + │ + ▼ +Severity Classification (priority label) + │ + ▼ +Persistence (MongoDB) + Publish (Redis Stream) + │ + ├──▶ SSE to browser (real-time log viewer) + │ + ▼ +Correlation Engine (pattern matching across events) + │ + ▼ +Alert Generation + │ + ├──▶ SSE to browser (real-time alert feed) + │ + ▼ +Alert Lifecycle (new → acknowledged → investigating → resolved) + │ + ▼ +Forensic Investigation (pivot queries, matched event drill-down) +``` + +Each concept builds on the previous one. Without normalization, correlation can't compare events across sources. Without severity classification, analysts can't prioritize. Without streaming, response time suffers. Without correlation, individual events are meaningless noise. + +## Industry Standards and Frameworks + +### OWASP + +While OWASP focuses on application security, several categories apply to the SIEM itself: +- **A01:2021 Broken Access Control** - The RBAC system with `@endpoint(roles=ADMIN)` prevents unauthorized access to admin functions. The `_prevent_self_action()` helper blocks admins from deactivating their own accounts. +- **A02:2021 Cryptographic Failures** - Argon2id for password hashing, HS256 JWTs with configurable expiration. The `SECRET_KEY` default of "change-me-in-production" is intentionally obvious. +- **A07:2021 Identification and Authentication Failures** - Timing-safe password verification, rate limiting on auth endpoints (10/minute), account deactivation support. + +### MITRE ATT&CK + +The scenario playbooks map directly to MITRE techniques: + +- **T1110.001 (Brute Force: Password Guessing)** - `brute_force_lateral.yml` simulates 20 SSH password attempts across common usernames +- **T1021.004 (Remote Services: SSH)** - Lateral movement via SSH key reuse after initial compromise +- **T1048.003 (Exfiltration Over Alternative Protocol: DNS)** - `data_exfiltration.yml` simulates DNS tunneling with high-entropy TXT queries +- **T1059.001 (Command and Scripting Interpreter: PowerShell)** - `phishing_c2.yml` includes encoded PowerShell download cradle execution +- **T1068 (Exploitation for Privilege Escalation)** - `privilege_escalation.yml` simulates kernel exploit compilation and execution +- **T1071.001 (Application Layer Protocol: Web)** - C2 beaconing over HTTPS with regular intervals + +Correlation rules store `mitre_tactic` and `mitre_technique` fields, and these propagate to generated alerts for analyst context. + +### CWE + +Common weakness enumerations relevant to what this project teaches: +- **CWE-778 (Insufficient Logging)** - The entire project is about solving this. Seven source types with format-specific normalization. +- **CWE-223 (Omission of Security-Relevant Information)** - The normalizer preserves raw event data alongside normalized fields. +- **CWE-779 (Logging of Excessive Data)** - Redis Streams use `maxlen` with approximate trimming (`STREAM_MAXLEN = 10000`) to prevent unbounded growth. +- **CWE-307 (Improper Restriction of Excessive Authentication Attempts)** - Rate limiting on auth endpoints plus correlation rules for brute force detection. + +## Real World Examples + +### Case Study 1: Target Corporation Breach (2013) + +Target's network was breached via a phishing email sent to an HVAC vendor. The attackers moved laterally to POS systems and installed RAM-scraping malware. Target's FireEye IDS detected the malware and generated alerts, but the SOC team failed to correlate those alerts with the broader attack chain. + +This is exactly the kind of failure a SIEM with proper correlation prevents. A sequence rule looking for "external access → credential theft → lateral movement → data access" would have connected the dots. The scenario playbook `brute_force_lateral.yml` in this project simulates a similar kill chain: external brute force → credential harvest → SSH lateral movement → database dump. + +### Case Study 2: DNS Tunneling in the APT34/OilRig Campaign + +APT34 (also known as OilRig) used DNS tunneling to exfiltrate data from compromised networks. They encoded stolen data in DNS query subdomains, sending it to attacker-controlled nameservers. The technique works because most firewalls allow outbound DNS traffic without deep inspection. + +The `data_exfiltration.yml` playbook simulates this exact technique. It generates high-entropy TXT queries to subdomains like `4d5a90000300000004000000ffff0000b800.data.exfil-tunnel.tk`. A threshold correlation rule on DNS query frequency, or an aggregation rule on distinct subdomains per destination domain, would detect this activity. + +### Case Study 3: Emotet/TrickBot Delivery Chain + +TA542's campaigns follow a consistent pattern: spear-phishing email → macro-enabled document → PowerShell download cradle → second-stage payload → C2 beaconing. The `phishing_c2.yml` playbook reproduces this chain event by event, from the initial proxy log of the phishing link click through to periodic C2 beacon heartbeats. + +This demonstrates why sequence correlation matters. Any individual event in the chain could be benign. Downloading an Excel file is normal. Running PowerShell is normal. Outbound HTTPS is normal. But the sequence of Excel → cmd.exe → PowerShell → download → C2 beaconing is not normal, and a sequence rule can catch it. + +## Testing Your Understanding + +Before moving to the architecture, make sure you can answer: + +1. Why does the normalizer preserve the raw event data in addition to normalized fields? What investigation scenario would require the raw data? +2. What's the difference between a threshold rule and an aggregation rule? Give an example attack that each would detect but the other wouldn't. +3. Why does `verify_password_timing_safe()` hash against a dummy value when the user doesn't exist? What information leak does this prevent? +4. If the correlation engine's cooldown is set to 300 seconds, what happens during a sustained 10-minute brute force? How many alerts would fire? +5. Why does the SSE generator send keepalive comments? What would happen in a production deployment behind Nginx without them? + +If these questions feel unclear, re-read the relevant sections. The implementation details will make more sense once these fundamentals click. + +## Further Reading + +**Essential:** +- [MITRE ATT&CK Framework](https://attack.mitre.org/) - The taxonomy this project's scenarios and rules are built around. Start with the "Enterprise" matrix. +- [Redis Streams documentation](https://redis.io/docs/data-types/streams/) - Understanding XADD, XREADGROUP, consumer groups, and acknowledgment is critical for understanding the streaming layer. + +**Deep dives:** +- [Sigma Rules](https://github.com/SigmaHQ/sigma) - An open standard for SIEM detection rules. Challenge 6 in the challenges doc suggests implementing Sigma rule support. +- [OWASP Password Storage Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html) - Why Argon2id was chosen over bcrypt, and the recommended parameter tuning. + +**Historical context:** +- Mandiant's APT1 Report (2013) - One of the first public threat intelligence reports, and a great example of the kind of investigation workflow a SIEM enables. +- NIST SP 800-92 (Guide to Computer Security Log Management) - The foundational document on why and how to collect, store, and analyze security logs. diff --git a/PROJECTS/intermediate/siem-dashboard/learn/02-ARCHITECTURE.md b/PROJECTS/intermediate/siem-dashboard/learn/02-ARCHITECTURE.md new file mode 100644 index 00000000..52207c8d --- /dev/null +++ b/PROJECTS/intermediate/siem-dashboard/learn/02-ARCHITECTURE.md @@ -0,0 +1,598 @@ +# System Architecture + +How the SIEM dashboard is designed, why each component exists, and how data moves through the system. + +## High Level Architecture + +``` +┌──────────────────────────────────────────────────────────────┐ +│ Browser │ +│ React 19 + TypeScript + Zustand + TanStack Query │ +│ SSE Listeners (EventSource) │ +└──────────────┬────────────────────────────┬──────────────────┘ + │ HTTP/JSON │ SSE + ▼ ▼ +┌──────────────────────────────────────────────────────────────┐ +│ Nginx Reverse Proxy │ +│ Rate limiting · Gzip · Static assets · SSE passthrough │ +│ proxy_buffering off (for /stream endpoints) │ +└──────────────┬────────────────────────────┬──────────────────┘ + │ │ + ▼ ▼ +┌──────────────────────────────────────────────────────────────┐ +│ Flask Backend │ +│ │ +│ ┌─────────┐ ┌──────────────┐ ┌────────────────────┐ │ +│ │ Routes │→ │ Controllers │→ │ Engine │ │ +│ │ (auth, │ │ (validation, │ │ (normalizer, │ │ +│ │ logs, │ │ business │ │ severity, │ │ +│ │ alerts │ │ logic) │ │ correlation) │ │ +│ │ rules) │ │ │ │ │ │ +│ └─────────┘ └──────────────┘ └────────────────────┘ │ +│ │ +│ ┌──────────────────────┐ ┌──────────────────────────┐ │ +│ │ Scenario Runner │ │ Correlation Engine │ │ +│ │ (daemon threads, │ │ (daemon thread, │ │ +│ │ playbook playback) │ │ XREADGROUP consumer) │ │ +│ └──────────────────────┘ └──────────────────────────┘ │ +└──────────┬──────────────────────────┬────────────────────────┘ + │ │ + ▼ ▼ +┌────────────────────┐ ┌────────────────────┐ +│ MongoDB 8.0 │ │ Redis 7 Alpine │ +│ │ │ │ +│ log_events │ │ siem:logs stream │ +│ alerts │ │ siem:alerts stream│ +│ correlation_rules │ │ rate limit keys │ +│ users │ │ │ +│ scenario_runs │ │ │ +└────────────────────┘ └────────────────────┘ +``` + +### Component Breakdown + +**Nginx Reverse Proxy** +Sits in front of everything. Handles TLS termination in production, rate limiting at the edge (`limit_req_zone` in `conf/nginx/nginx.conf:34-36`), gzip compression, and static asset caching. The critical detail is the SSE passthrough configuration. Regular API endpoints use buffered proxying, but `/stream` endpoints need `proxy_buffering off` and a 3600s read timeout (`conf/nginx/prod.nginx:30-43`). Without this, Nginx buffers SSE events and the browser gets nothing until the buffer fills. + +**Flask Backend** +Application factory pattern in `backend/app/__init__.py`. Creates the Flask app, wires up MongoDB and Redis connections, registers blueprints, initializes rate limiting, creates Redis consumer groups, cleans up orphaned scenario runs, and starts the correlation engine. That startup order matters. The consumer groups must exist before the correlation engine tries to read from them. + +**MongoDB** +Primary data store for all persistent documents. MongoEngine ODM maps Python classes to collections. Five main collections: `log_events`, `alerts`, `correlation_rules`, `users`, `scenario_runs`. Each has specific indexes defined in the model `meta` dict for query performance. + +**Redis** +Two roles. First, it powers the streaming pipeline. Two Redis Streams (`siem:logs` and `siem:alerts`) handle the pub/sub fanout between log ingestion, the correlation engine, and SSE endpoints. Second, it backs the rate limiter (`flask-limiter` uses Redis as its storage backend via the `REDIS_URL` config). + +**React Frontend** +Single page app with React 19, TypeScript, and a clean separation between data fetching (TanStack Query hooks in `frontend/src/api/hooks/`) and UI state (Zustand stores in `frontend/src/core/stores/`). Real time updates come through two SSE connections managed by `useLogStream` and `useAlertStream` hooks. + +## Data Flow + +### Log Ingestion Pipeline + +This is the core pipeline. Every log event, whether from the scenario runner or an external POST, follows the same path: + +``` +1. Raw event arrives + POST /v1/logs/ingest (or ScenarioRunner._emit_event) + │ +2. Normalize (app/engine/normalizer.py) + │ Dispatches to source_type-specific normalizer + │ Extracts common fields, preserves raw payload + │ +3. Classify severity (app/engine/severity.py) + │ Event type lookup → regex pattern matching → default to info + │ +4. Persist to MongoDB (app/models/LogEvent.py) + │ LogEvent.create_event() → document saved with all fields + │ +5. Publish to Redis Stream (app/core/streaming.py) + │ XADD siem:logs {payload: JSON} + │ Maxlen ~10000 (approximate trim) + │ + ├──→ 6a. Correlation Engine reads via XREADGROUP + │ │ Evaluates all enabled rules against event + │ │ If rule fires → create Alert → XADD siem:alerts + │ │ ACK message after processing + │ │ + │ └──→ SSE /v1/alerts/stream (browser picks up new alerts) + │ + └──→ 6b. SSE /v1/logs/stream + │ XREAD with blocking, yields to EventSource + └── Browser pushes to Zustand StreamStore +``` + +The important thing here is that steps 4 and 5 happen atomically from the controller's perspective. The event is persisted before it hits the stream. If Redis is down, the event is still in MongoDB. If MongoDB is down, nothing gets published because `create_event()` would throw first. + +### Alert Lifecycle + +Alerts have a state machine with five statuses: + +``` +new → acknowledged → investigating → resolved + → false_positive +``` + +When the correlation engine fires a rule (`app/engine/correlation.py:_process_event`), it calls `Alert.create_from_rule()` which both saves the alert to MongoDB and publishes it to the `siem:alerts` Redis Stream. The alert document references the matched event IDs so analysts can drill into what triggered the alert. + +Status transitions happen through `PATCH /v1/alerts//status` and are tracked with timestamps. The `acknowledged_by` field records which analyst claimed the alert, and `resolved_at` marks closure time. + +### Authentication Flow + +``` +Register/Login + │ + ▼ +POST /v1/auth/register or POST /v1/auth/login + │ │ + ▼ ▼ +hash_password() verify_password_timing_safe() +(Argon2id) (constant time, dummy hash for missing users) + │ │ + ▼ ▼ +User.create_user() Validate credentials + │ │ + └───────┬───────────────────┘ + ▼ + create_access_token() + JWT with sub=user_id, username, role, exp + │ + ▼ + Return {access_token, token_type: "bearer"} + │ + ▼ + Frontend stores in Zustand (persisted to localStorage) + Axios interceptor attaches to every request +``` + +Each subsequent request hits the `endpoint()` decorator which extracts the Bearer token, decodes the JWT, loads the user from MongoDB, and attaches it to Flask's `g.current_user`. The decorator also handles role gating. Pass `roles=["admin"]` and non-admins get a 403. + +## Design Patterns + +### Application Factory + +**Where it lives:** `backend/app/__init__.py` + +The `create_app()` function builds the Flask app from scratch every time it's called. This isn't just a Flask convention. It solves real problems: test isolation (each test gets a fresh app), configuration flexibility (swap `.env` files between dev/prod), and import order issues (extensions initialize after app config is set). + +The initialization order is deliberate: + +1. Config loading (from env vars via Pydantic) +2. CORS setup +3. MongoDB and Redis connections +4. Error handlers +5. Rate limiter +6. Blueprint registration +7. Consumer group creation +8. Orphan scenario cleanup +9. Correlation engine start + +If you move step 7 after step 9, the correlation engine will crash trying to read from a consumer group that doesn't exist yet. + +### Decorator Stack Pattern + +**Where it lives:** `backend/app/core/decorators/` + +Every route handler uses the same decorator stack: `@endpoint` → `@S` → `@R`. This is a composable pipeline: + +```python +# backend/app/routes/logs.py:32-39 +@logs_bp.post("/ingest") +@endpoint(auth_required=False) +@S(LogIngestRequest) +@R(status=201) +def ingest_log() -> Any: + return log_ctrl.ingest_log() +``` + +`@endpoint` handles JWT extraction, user loading, role enforcement, and error boundaries. `@S` (Schema) validates request data with Pydantic and stores the result on `g.validated`. `@R` (Response) auto-serializes the return value into JSON with the right status code. + +The decorators execute outside-in: endpoint runs first (auth check), then S (validation), then the function body, then R (serialization). If auth fails, validation never runs. If validation fails, the controller never executes. This fail-fast approach keeps controller code clean. + +**Trade-offs:** +The decorator stack is concise but can be confusing to debug. Stack traces go through multiple wrapper layers. If you add a new decorator, ordering matters and gets it wrong silently. + +### Registry Pattern for Normalizers + +**Where it lives:** `backend/app/engine/normalizer.py` + +Each source type (firewall, IDS, auth, endpoint, DNS, proxy, generic) has its own normalizer function registered via the `@_register` decorator: + +```python +# backend/app/engine/normalizer.py:16-22 +def _register(source_type: SourceType) -> Callable[[NormalizerFn], NormalizerFn]: + def decorator(fn: NormalizerFn) -> NormalizerFn: + NORMALIZERS[source_type.value] = fn + return fn + return decorator +``` + +The `normalize()` dispatcher looks up the right function from the `NORMALIZERS` dict and falls back to `_normalize_generic` for unknown types. Adding a new source type means writing one function and adding the decorator. No switch statements, no if/elif chains, no modification of the dispatch logic. + +### Thread-safe State with Locks + +**Where it lives:** `backend/app/engine/correlation.py` (`CorrelationState` class) + +The correlation engine runs on a daemon thread but shares state structures (sliding windows, cooldown timestamps) that could be accessed during rule testing from the main Flask thread. Every method on `CorrelationState` acquires `self._lock` before touching `_windows` or `_cooldowns`. This prevents data races but means the correlation engine can't process two events in parallel. For the throughput this project targets, that's fine. + +## Layer Separation + +``` +┌────────────────────────────────────────────────────┐ +│ Routes Layer (app/routes/) │ +│ HTTP concerns only: URL mapping, rate limits │ +│ Does NOT: query databases, process data │ +└───────────────────────┬────────────────────────────┘ + ▼ +┌────────────────────────────────────────────────────┐ +│ Controller Layer (app/controllers/) │ +│ Business logic: orchestrate models and engine │ +│ Does NOT: parse requests, format responses │ +└───────────────────────┬────────────────────────────┘ + ▼ +┌────────────────────────────────────────────────────┐ +│ Model Layer (app/models/) │ +│ Data access: MongoEngine documents, queries │ +│ Does NOT: know about HTTP, Flask request/response│ +└───────────────────────┬────────────────────────────┘ + ▼ +┌────────────────────────────────────────────────────┐ +│ Engine Layer (app/engine/) │ +│ Domain logic: normalization, severity, correlation│ +│ Does NOT: persist data, know about Flask │ +└────────────────────────────────────────────────────┘ +``` + +### What Lives Where + +**Routes** (`app/routes/`): Blueprint definitions, URL patterns, decorator stacks. Each route function is a thin wrapper that calls the corresponding controller function. Routes import from controllers and schemas, never from models directly. + +**Controllers** (`app/controllers/`): Business logic coordination. The controller for log ingestion (`log_ctrl.ingest_log`) calls the normalizer, the severity classifier, the model's `create_event`, and the streaming publisher. Controllers access `g.validated` for input and `g.current_user` for auth context. + +**Models** (`app/models/`): MongoEngine document definitions with query methods. `BaseDocument` provides shared functionality like `get_by_id`, `paginate`, and auto-updating `updated_at` timestamps. Models can call other models (Alert references LogEvent) but never import from routes or controllers. + +**Engine** (`app/engine/`): Pure domain logic. The normalizer, severity classifier, and correlation engine live here. The correlation engine is the only component that reaches into both the streaming layer (to read events) and the model layer (to create alerts). This is a pragmatic trade-off to keep the daemon thread self-contained. + +## Data Models + +### LogEvent + +```python +# backend/app/models/LogEvent.py +class LogEvent(BaseDocument): + meta = { + "collection": "log_events", + "ordering": ["-timestamp"], + "indexes": [ + "timestamp", "source_type", "severity", + "source_ip", "dest_ip", "username", + "hostname", "event_type", "scenario_run_id", + ], + } + timestamp = DateTimeField() + source_type = StringField(required=True) # firewall, ids, auth, etc. + source_ip = StringField() + dest_ip = StringField() + source_port = IntField() + dest_port = IntField() + severity = StringField(default="info") + event_type = StringField() # login_failure, port_scan, etc. + raw = DictField() # original payload, untouched + normalized = DictField() # source-type-specific fields + hostname = StringField() + username = StringField() + scenario_run_id = ObjectIdField() # links to ScenarioRun if simulated +``` + +Nine indexes cover the query patterns used by the log viewer, pivot searches, and dashboard aggregations. The `raw` field preserves the original event exactly as submitted. The `normalized` field holds source-type-specific fields extracted by the normalizer. This dual storage means you can always go back to the original data if the normalizer had a bug. + +### Alert + +```python +# backend/app/models/Alert.py +class Alert(BaseDocument): + rule_id = ObjectIdField(required=True) + rule_name = StringField(required=True) + severity = StringField(required=True) + title = StringField(required=True) # "{rule_name} [{group_value}]" + matched_event_ids = ListField(ObjectIdField()) # references to LogEvent docs + matched_event_count = IntField(default=0) + group_value = StringField() # the IP, username, etc. that grouped + status = StringField(default="new") + mitre_tactic = StringField() + mitre_technique = StringField() + acknowledged_by = StringField() + acknowledged_at = DateTimeField() + resolved_at = DateTimeField() +``` + +Alerts link back to both the rule that generated them (`rule_id`) and the specific events that matched (`matched_event_ids`). The `get_with_events()` method loads the referenced LogEvent documents for the alert detail view. This is a manual join since MongoDB doesn't do relational joins, but the list of IDs is bounded by the correlation window size, so it's never thousands of documents. + +### CorrelationRule + +```python +# backend/app/models/CorrelationRule.py +class CorrelationRule(BaseDocument): + name = StringField(required=True, unique=True) + rule_type = StringField(required=True) # threshold, sequence, aggregation + conditions = DictField(required=True) # type-specific config + severity = StringField(required=True) + enabled = BooleanField(default=True) + mitre_tactic = StringField() + mitre_technique = StringField() +``` + +The `conditions` field is a flexible dict whose shape depends on `rule_type`. For threshold rules it contains `event_filter`, `threshold`, `window_seconds`, and `group_by`. For sequence rules it has `steps` (an ordered list of event filters). For aggregation rules it adds `aggregation_field` for counting distinct values. Validation of these shapes happens in the Pydantic schemas (`backend/app/schemas/rule.py:60-82`) using a `@model_validator` that dispatches to the correct condition schema based on `rule_type`. + +## Security Architecture + +### Threat Model + +What the platform defends against: + +1. **Credential stuffing on the login endpoint.** Rate limiting at both Nginx (3r/s for auth endpoints) and Flask (10/minute via `flask-limiter`) makes brute force impractical. The `verify_password_timing_safe` function prevents username enumeration through timing differences. + +2. **Unauthorized access to SIEM data.** JWT-based authentication on every API endpoint (except `/v1/logs/ingest` and public auth routes). Role-based access control gates admin operations. The `endpoint()` decorator enforces this uniformly. + +3. **Privilege escalation via role manipulation.** The `update_role` controller in `admin_ctrl.py:36-43` checks `User.count_admins()` before allowing demotion. You can't demote the last admin. Admins also can't deactivate or delete their own accounts. + +4. **SSE token leakage.** SSE connections can't use the Authorization header (browser limitation with EventSource), so tokens go in the query string. The `extract_bearer_token()` function in `app/core/auth.py:97-102` checks the header first, then falls back to `request.args.get("token")`. This is a known trade-off. The token appears in server access logs but is transmitted over HTTPS. + +What's out of scope: +Network-level attacks (handled by infrastructure), database injection (MongoEngine parameterizes queries), XSS (React escapes by default, plus security headers from Nginx). + +### Defense in Depth + +``` +Nginx Layer +├── Rate limiting (10r/s API, 3r/s auth) +├── Connection limits (50 per IP in prod) +├── Security headers (X-Frame-Options, X-Content-Type-Options, etc.) +└── Request size limits (10MB max body) + │ +Flask Layer +├── flask-limiter (moving window, 200/min default, 10/min auth) +├── Pydantic validation (all inputs validated before processing) +├── JWT verification (expiry, required claims) +└── Role enforcement (decorator-based RBAC) + │ +Application Layer +├── Argon2id password hashing +├── Constant-time password verification +├── Last-admin protection +└── Self-action prevention (can't deactivate your own account) +``` + +## Storage Strategy + +### MongoDB: Persistent State + +All documents that need to survive restarts. Log events, alerts, correlation rules, users, scenario runs. MongoEngine provides the ODM layer with type coercion and validation. + +The `BaseDocument` class (`app/models/Base.py`) adds `created_at` and `updated_at` to every document and overrides `save()` to auto-update the timestamp. The `paginate()` class method provides consistent offset-based pagination across all collections. + +Index strategy: every field used in a filter or sort gets an index. LogEvent has nine indexes, which is aggressive but appropriate for a query-heavy dashboard. Write performance takes a small hit, but reads (which dominate in a SIEM) stay fast. + +### Redis: Ephemeral Streams and Rate Limits + +Two Redis Streams with approximate maxlen of 10,000 entries each. The streams are ephemeral. Restarting Redis loses unprocessed messages, but since events are persisted in MongoDB first, the only impact is that the correlation engine might miss some events during the restart window. For a learning project, this is an acceptable trade-off. Production SIEMs would use persistent queues. + +Rate limit counters also live in Redis. The `moving-window` strategy in `flask-limiter` stores sliding window counters keyed by client IP. + +## Configuration + +### Environment Variables + +All configuration flows through `backend/app/config.py` using Pydantic Settings: + +```python +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + case_sensitive=False, + ) + MONGO_URI: str = "mongodb://mongo:27017/siem" + REDIS_URL: str = "redis://redis:6379/0" + SECRET_KEY: str = "change-me-in-production" + JWT_EXPIRATION_HOURS: int = 24 + CORRELATION_COOLDOWN_SECONDS: int = 300 + RATELIMIT_DEFAULT: str = "200/minute" + RATELIMIT_AUTH: str = "10/minute" + # ... ~40 more settings with defaults +``` + +Every setting has a sensible default, so the project works out of the box with just `docker compose up`. But the defaults are explicitly not secure for production. The `SECRET_KEY` default is `"change-me-in-production"` and Redis has no password in dev mode. + +### Development vs Production + +**Development** (`dev.compose.yml`): Hot reload on both frontend and backend. No Nginx auth rate limiting. Redis without a password. MongoDB port exposed on host. Frontend runs as a Vite dev server with HMR. + +**Production** (`compose.yml`): Gunicorn with 4 workers. Frontend pre-built and served as static files from Nginx. Redis requires `REDIS_PASSWORD`. Nginx adds security headers, connection limits, and aggressive caching for static assets. Docker resource limits are set (backend gets 2 CPUs, 1GB RAM). + +## Performance Considerations + +### Bottlenecks + +**Correlation engine is single-threaded.** The `CorrelationEngine._run()` loop processes events sequentially. Under heavy load (hundreds of events per second), the consumer group will accumulate a backlog. The `STREAM_READ_COUNT` setting (default 10) and `STREAM_BLOCK_MS` (default 2000) control the batch size and blocking behavior. + +**MongoDB aggregations for the dashboard.** The `timeline_aggregation`, `severity_breakdown`, and `top_sources` methods all run aggregation pipelines. With millions of log events, these become slow. The `$dateTrunc` grouping in the timeline pipeline is particularly expensive without a compound index on `(timestamp, severity)`. + +**SSE connections hold threads.** Each SSE client holds a Gunicorn worker thread open for the entire connection duration. With 4 workers and 3 SSE clients, only 1 worker is available for regular API requests. Production would need either more workers or an async SSE solution. + +### Optimizations Already Present + +**Rule caching.** The correlation engine caches enabled rules for `CORRELATION_RULE_CACHE_SECONDS` (default 30s) to avoid hitting MongoDB on every event. See `CorrelationEngine._get_rules()`. + +**Approximate stream trimming.** `XADD` with `approximate=True` lets Redis trim the stream lazily rather than on every write. Slightly exceeds the 10,000 maxlen but avoids the per-write trim overhead. + +**Sliding window cleanup.** `CorrelationState.get_window()` evicts expired entries every time a window is read, keeping memory bounded without a separate cleanup thread. + +## Deployment Architecture + +### Docker Compose Topology + +``` + ┌──────────┐ + :8431 ────→│ Nginx │ + └────┬─────┘ + │ + ┌─────────┴─────────┐ + │ │ + ┌────▼────┐ ┌────▼────┐ + │ Backend │ │ Static │ + │ (Flask) │ │ (built │ + │ :5000 │ │ React) │ + └────┬────┘ └─────────┘ + │ + ┌────────┴────────┐ + │ │ +┌────▼────┐ ┌──────▼─────┐ +│ MongoDB │ │ Redis │ +│ :27017 │ │ :6379 │ +└─────────┘ └────────────┘ +``` + +Two Docker networks isolate traffic. The `frontend` network connects Nginx to the backend. The `backend` network connects the backend to MongoDB and Redis. MongoDB and Redis are not directly accessible from the frontend network. + +Resource limits in production: +- Nginx: 1 CPU, 256MB RAM +- Backend: 2 CPUs, 1GB RAM +- MongoDB: 1 CPU, 512MB RAM +- Redis: 0.5 CPU, 256MB RAM + +### Health Checks + +Every service has a health check. The backend exposes `/health` which returns `"1"` (defined in `app/__init__.py`). MongoDB uses `mongosh --eval "db.adminCommand('ping')"`. Redis uses `redis-cli ping`. Nginx depends on the backend being healthy before it starts accepting traffic. + +## Design Decisions + +### MongoDB over PostgreSQL + +MongoDB was chosen because log events are semi-structured. Different source types have different fields (firewall events have `action` and `protocol`, DNS events have `query` and `query_type`). Storing this in a relational schema would mean either a wide sparse table or a separate table per source type. MongoDB's flexible documents with the `DictField` for `raw` and `normalized` handle this naturally. + +Trade-off: no transactional joins. The alert detail view requires a manual lookup of matched events by ID. With PostgreSQL, this would be a single JOIN query. + +### Redis Streams over Kafka or RabbitMQ + +Redis Streams provide enough pub/sub for this use case without adding another infrastructure dependency. The consumer group semantics (`XREADGROUP`, `XACK`) give exactly-once processing for the correlation engine. SSE endpoints use plain `XREAD` (no consumer group) since they're just tailing the stream for display. + +Trade-off: no persistence guarantees, no multi-node replication, limited backpressure. A production SIEM would need something more robust. + +### JWT over Sessions + +Stateless auth means the backend doesn't need to store sessions in Redis or MongoDB. Each request carries its own proof of identity. The frontend stores the token in Zustand (persisted to localStorage via the `persist` middleware in `frontend/src/core/stores/auth.store.ts`). + +Trade-off: you can't invalidate a JWT before it expires. If a user's account is deactivated, they can still make requests until the token's `exp` claim passes. The `endpoint()` decorator mitigates this by loading the user from MongoDB on every request and checking `is_active`. + +### Pydantic for Request Validation + +Every endpoint validates input through Pydantic schemas via the `@S` decorator. This catches bad data at the boundary before it reaches controller logic. The correlation rule schemas are particularly interesting. The `RuleCreateRequest` uses a `@model_validator` to dispatch condition validation based on `rule_type`, so threshold rules validate differently from sequence rules. + +Trade-off: Pydantic adds import time and a layer of indirection. Simple endpoints that just take an ID don't need validation but still go through the decorator stack. + +## Error Handling Strategy + +### Error Hierarchy + +```python +# backend/app/core/errors.py +AppError (500) +├── NotFoundError (404) +├── ValidationError (422, includes field-level details) +├── AuthenticationError (401) +├── ForbiddenError (403) +└── ConflictError (409) +``` + +All custom errors extend `AppError`, which carries a `status_code` and `message`. The `register_error_handlers()` function in `errors.py` attaches a single Flask error handler for `AppError` that serializes any subclass into a consistent JSON response: `{"error": "ErrorClassName", "message": "..."}`. + +The `endpoint()` decorator adds a catch-all for unexpected exceptions. If a controller raises something that isn't an `AppError`, the decorator logs the traceback via structlog and returns a generic 500. This prevents stack traces from leaking to the client. + +### Frontend Error Handling + +The frontend mirrors this with `ApiError` in `frontend/src/core/lib/errors.ts`. The Axios response interceptor transforms HTTP errors into typed `ApiError` instances with codes like `AUTHENTICATION_ERROR`, `VALIDATION_ERROR`, etc. The React Query cache has global error handlers that show toast notifications for background query failures. + +## Extensibility + +### Adding a New Log Source Type + +1. Add the source type to `SourceType` enum in `backend/app/models/LogEvent.py` +2. Write a normalizer function in `backend/app/engine/normalizer.py`: + +```python +@_register(SourceType.YOUR_NEW_TYPE) +def _normalize_your_type(raw: dict[str, Any]) -> dict[str, Any]: + return { + "your_field": raw.get("your_field"), + # ... extract source-specific fields + } +``` + +3. Optionally add severity patterns in `backend/app/engine/severity.py` + +That's it. The registry pattern means no other code changes are needed. The ingest endpoint, SSE streaming, correlation engine, and frontend log viewer all work with any source type. + +### Adding a New Correlation Rule Type + +1. Add the type to `RuleType` enum in `backend/app/models/CorrelationRule.py` +2. Add a Pydantic conditions schema in `backend/app/schemas/rule.py` +3. Write the evaluator function in `backend/app/engine/correlation.py`: + +```python +def _evaluate_your_type(rule, event_data, state, rule_id, group_key): + # Your evaluation logic + # Return EvaluationResult if fired, None otherwise +``` + +4. Register it in the `evaluators` dict inside `evaluate_rule()` + +### Adding a New API Endpoint + +1. Create or update a schema in `backend/app/schemas/` +2. Write the controller function in `backend/app/controllers/` +3. Add the route in `backend/app/routes/` with the decorator stack +4. Add the frontend hook in `frontend/src/api/hooks/` +5. Add the endpoint path to `frontend/src/config.ts` + +## Limitations + +**No horizontal scaling.** The correlation engine uses in-memory state (`CorrelationState`). Running multiple backend instances means each instance has its own sliding windows and cooldowns. Events would be split across instances via the consumer group, but the state wouldn't be shared. Fixing this would require moving correlation state to Redis. + +**No event deduplication.** If the same event is ingested twice (network retry, for example), it gets stored twice and may trigger correlation rules twice. A production system would hash the raw event and check for duplicates before persisting. + +**No log retention policy.** Events accumulate in MongoDB forever. The dashboard aggregations will slow down as the collection grows. A TTL index on `timestamp` or a periodic cleanup job would fix this. + +**Single consumer for correlation.** The `CONSUMER_NAME` is hardcoded to `"engine-1"`. Adding more consumers would require partitioning logic to prevent duplicate alert generation. + +## Key Files Reference + +Backend core: +- `backend/app/__init__.py` - Application factory, startup sequence +- `backend/app/config.py` - All settings via Pydantic +- `backend/app/core/auth.py` - JWT and password operations +- `backend/app/core/streaming.py` - Redis Streams pub/sub and SSE +- `backend/app/core/decorators/` - endpoint, S, R decorator stack +- `backend/app/engine/correlation.py` - Correlation engine and rule evaluation +- `backend/app/engine/normalizer.py` - Log normalization registry +- `backend/app/engine/severity.py` - Severity classification +- `backend/app/scenarios/runner.py` - Scenario playback threads + +Frontend core: +- `frontend/src/config.ts` - API endpoints, query keys, routes +- `frontend/src/core/lib/api.ts` - Axios client with interceptors +- `frontend/src/core/stores/auth.store.ts` - Auth state with persistence +- `frontend/src/core/stores/stream.store.ts` - SSE event buffer +- `frontend/src/api/hooks/useEventStream.ts` - SSE connection management + +Infrastructure: +- `dev.compose.yml` - Development Docker setup +- `compose.yml` - Production Docker setup +- `conf/nginx/nginx.conf` - Nginx main config with rate limits +- `conf/nginx/prod.nginx` - Production server block with SSE passthrough + +## Next Steps + +Now that you understand the architecture: +1. Read [03-IMPLEMENTATION.md](./03-IMPLEMENTATION.md) for a code-level walkthrough +2. Try modifying the normalizer to add a new source type and see the registry pattern in action diff --git a/PROJECTS/intermediate/siem-dashboard/learn/03-IMPLEMENTATION.md b/PROJECTS/intermediate/siem-dashboard/learn/03-IMPLEMENTATION.md new file mode 100644 index 00000000..4d3fe722 --- /dev/null +++ b/PROJECTS/intermediate/siem-dashboard/learn/03-IMPLEMENTATION.md @@ -0,0 +1,1296 @@ +# Implementation Walkthrough + +This document walks through the actual code in the SIEM dashboard, file by file. We'll trace how a log event enters the system, gets normalized, classified, published to a stream, correlated against rules, and surfaced as an alert in the browser. Each section references real files, real functions, and explains why the code is structured the way it is. + +If you haven't read [02-ARCHITECTURE.md](./02-ARCHITECTURE.md) yet, do that first. This document assumes you understand the high level component layout and data flow. + +## Application Bootstrap + +### The Factory Pattern + +`app/__init__.py` contains `create_app()`, the Flask application factory. This function builds the entire backend in a specific order that matters. + +```python +def create_app() -> Flask: + app = Flask(__name__) + config = get_settings() + app.config.from_mapping(config.flask_config()) + + # 1. Extensions first (database + cache connections) + init_extensions(app) + + # 2. Error handlers before anything that might raise + register_error_handlers(app) + + # 3. Rate limiter needs app context + init_rate_limiter(app) + + # 4. Blueprints register routes + register_blueprints(app) + + # 5. Streaming infrastructure + ensure_consumer_group() + + # 6. Clean up orphaned scenario runs from previous crashes + cleanup_orphaned_runs() + + # 7. Start correlation engine last (needs everything else ready) + start_correlation_engine(app) + + return app +``` + +The ordering is deliberate. You can't register error handlers after blueprints start throwing errors. You can't start the correlation engine before MongoDB and Redis connections exist. And you need consumer groups created before the correlation engine tries to read from them. + +A common mistake when building Flask apps: initializing things at module import time instead of inside the factory. That breaks testing (you can't create multiple app instances with different configs) and it breaks Gunicorn (the fork model means each worker needs its own connections). Everything here happens inside `create_app()`, which means each Gunicorn worker gets its own setup. + +### Configuration + +`app/config.py` uses Pydantic's `BaseSettings` to load configuration from environment variables with sensible defaults. The settings class has around 60 values covering everything from MongoDB URIs to correlation engine tuning. + +```python +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env", extra="ignore") + + # Core + SECRET_KEY: str = "change-me-in-production" + DEBUG: bool = False + + # MongoDB + MONGODB_HOST: str = "mongodb://mongo:27017/siem" + + # Redis + REDIS_URL: str = "redis://redis:6379/0" + + # Correlation + CORRELATION_COOLDOWN_SECONDS: int = 300 + STREAM_READ_COUNT: int = 10 + STREAM_BLOCK_MS: int = 2000 + + # Rate limiting + RATELIMIT_DEFAULT: str = "200 per minute" + RATELIMIT_AUTH: str = "10 per minute" +``` + +The `flask_config()` method on Settings translates these into Flask's config format. Notice the `SECRET_KEY` default. That string is intentionally obvious so nobody accidentally ships it to production. Pydantic will override it from the environment or `.env` file automatically. + +The `get_settings()` function uses `@lru_cache` to create a singleton. This means the settings are parsed once (with all the Pydantic validation) and then reused. Without the cache, every access to settings would re-parse environment variables and re-validate. + +### Extensions + +`app/extensions.py` initializes MongoEngine and Redis as module-level singletons: + +```python +db = MongoEngine() +redis_client: Redis | None = None + +def init_extensions(app: Flask) -> None: + db.init_app(app) + + global redis_client + redis_client = Redis.from_url( + app.config["REDIS_URL"], + decode_responses=True, + ) +``` + +MongoEngine uses Flask's `init_app` pattern (lazy initialization). Redis is simpler: just create a connection from the URL. The `decode_responses=True` parameter means Redis returns Python strings instead of bytes. This matters everywhere downstream because you'll be JSON-serializing Redis data and `bytes` objects don't serialize cleanly. + +## Log Ingestion Pipeline + +This is the core pipeline. A raw event enters via HTTP POST, gets transformed, stored, published, correlated, and potentially triggers an alert. Let's trace the full path. + +### Step 1: HTTP Entry Point + +`app/routes/log_routes.py` defines the ingestion endpoint: + +```python +@bp.post("/ingest") +@S(body=LogIngestSchema) +@R(status_code=201) +def ingest_log(body: LogIngestSchema): + return log_ctrl.ingest_event(body) +``` + +Three decorators, each with a specific job. `@S(body=LogIngestSchema)` validates the request body against a Pydantic schema. `@R(status_code=201)` serializes the return value and sets the response code. The route itself is just one line of business logic delegation. + +Note this endpoint has no `@endpoint()` decorator, which means it's unauthenticated. This is intentional. In a real deployment, log sources (firewalls, servers, agents) send events to the SIEM without user credentials. They'd authenticate via API keys or network-level controls, but for this project the ingestion endpoint is open. + +### Step 2: Schema Validation + +`app/schemas/log_schemas.py` defines what a valid ingestion request looks like: + +```python +class LogIngestSchema(BaseModel): + source_type: SourceType + event_type: str + timestamp: datetime | None = None + source_ip: str | None = None + source_port: int | None = None + dest_ip: str | None = None + dest_port: int | None = None + protocol: str | None = None + username: str | None = None + hostname: str | None = None + message: str | None = None + # ... additional optional fields per source type +``` + +`SourceType` is a string enum: `firewall`, `ids`, `auth`, `endpoint`, `dns`, `proxy`, `generic`. Pydantic validates the enum value automatically. If you POST `"source_type": "foo"`, you get a 422 with a clear error message before any business logic runs. + +Most fields are optional because different source types provide different data. A firewall log has `source_ip` and `dest_ip` but no `username`. An auth log has `username` but might not have `dest_ip`. The normalizer handles making sense of whatever fields are present. + +### Step 3: Controller Logic + +`app/controllers/log_ctrl.py` orchestrates the ingestion: + +```python +def ingest_event(body: LogIngestSchema) -> dict: + raw = body.model_dump(exclude_none=True) + + # Normalize: common fields + source-specific extraction + normalized_data = normalize(raw) + + # Classify severity based on event type and content + severity = classify_severity(raw) + + # Build and save the document + event = LogEvent( + source_type=raw["source_type"], + event_type=raw["event_type"], + severity=severity, + timestamp=raw.get("timestamp", datetime.utcnow()), + source_ip=raw.get("source_ip"), + dest_ip=raw.get("dest_ip"), + # ... remaining fields + raw=raw, + normalized=normalized_data, + ) + event.save() + + # Publish to Redis Stream for real-time consumers + publish_log_event(event) + + return serialize_document(event) +``` + +Three operations happen in sequence: normalize, classify, persist. Then the event is published to the Redis Stream asynchronously. The `raw` dict is preserved on the document alongside the `normalized` dict. This dual storage is important for forensics, as covered in [01-CONCEPTS.md](./01-CONCEPTS.md). + +The controller calls `model_dump(exclude_none=True)` on the Pydantic schema to strip out fields that weren't provided. This prevents storing a bunch of `null` values in MongoDB and keeps the raw payload clean. + +### Step 4: Normalization + +`app/engine/normalizer.py` transforms raw events into a consistent structure. The registry pattern here is worth studying: + +```python +NORMALIZERS: dict[str, Callable] = {} + +def _register(source_type: SourceType): + """Decorator to register a normalizer function for a source type.""" + def decorator(fn: Callable) -> Callable: + NORMALIZERS[source_type.value] = fn + return fn + return decorator +``` + +Each source type gets its own normalizer function: + +```python +@_register(SourceType.FIREWALL) +def _normalize_firewall(data: dict) -> dict: + return { + "action": data.get("action"), + "protocol": data.get("protocol"), + "bytes_sent": data.get("bytes_sent", 0), + "bytes_received": data.get("bytes_received", 0), + } + +@_register(SourceType.AUTH) +def _normalize_auth(data: dict) -> dict: + return { + "auth_method": data.get("auth_method"), + "result": data.get("result"), + "failure_reason": data.get("failure_reason"), + } + +@_register(SourceType.DNS) +def _normalize_dns(data: dict) -> dict: + return { + "query": data.get("query"), + "query_type": data.get("query_type"), + "response_code": data.get("response_code"), + } +``` + +The dispatch function ties it together: + +```python +def normalize(data: dict) -> dict: + source_type = data.get("source_type", "generic") + normalizer = NORMALIZERS.get(source_type, NORMALIZERS["generic"]) + return normalizer(data) +``` + +This pattern makes adding new log sources trivial. Write a function, slap the `@_register` decorator on it, and you're done. No switch statements, no if/elif chains, no modification to existing code. The dispatcher looks up the function from the dict and calls it. + +One thing to note: if a source type isn't recognized, it falls back to the `generic` normalizer, which just grabs the `message` field. This is a design choice. Failing silently on unknown source types means the system keeps ingesting even if it encounters something unexpected. In production you'd want to log a warning, but dropping events is worse than storing them with minimal normalization. + +### Step 5: Severity Classification + +`app/engine/severity.py` assigns a severity level to each event. It uses a two tier approach: + +```python +HIGH_SEVERITY_EVENT_TYPES = frozenset({ + "privilege_escalation", "data_exfiltration", + "c2_communication", "reverse_shell", +}) + +MEDIUM_SEVERITY_EVENT_TYPES = frozenset({ + "login_failure", "port_scan", + "firewall_deny", "ids_alert", +}) + +CRITICAL_PATTERNS = [ + re.compile(r"privilege.?escalat", re.IGNORECASE), + re.compile(r"ransomware", re.IGNORECASE), + re.compile(r"c2.?beacon", re.IGNORECASE), + re.compile(r"data.?exfil", re.IGNORECASE), +] + +HIGH_PATTERNS = [ + re.compile(r"brute.?force", re.IGNORECASE), + re.compile(r"lateral.?movement", re.IGNORECASE), + re.compile(r"reverse.?shell", re.IGNORECASE), + re.compile(r"credential.?dump", re.IGNORECASE), +] +``` + +The `classify_severity()` function checks the event type first (fast O(1) frozen set lookup), then falls through to regex matching on concatenated text fields: + +```python +def classify_severity(data: dict) -> str: + event_type = data.get("event_type", "") + + # Fast path: check event type directly + if event_type in HIGH_SEVERITY_EVENT_TYPES: + return "high" + if event_type in MEDIUM_SEVERITY_EVENT_TYPES: + return "medium" + + # Slow path: regex against content + searchable = _build_searchable_text(data) + + for pattern in CRITICAL_PATTERNS: + if pattern.search(searchable): + return "critical" + for pattern in HIGH_PATTERNS: + if pattern.search(searchable): + return "high" + # ... medium and low patterns + + return "info" # Default +``` + +The `_build_searchable_text()` helper concatenates `event_type`, `message`, and relevant normalized fields (like `signature_name` from IDS events or `command_line` from endpoint events) into a single string. This means a PowerShell command line containing "lateral movement" gets classified as high severity even if the event_type is just "process_execution". + +The frozen sets are important. Python's `in` operator on a frozenset is O(1) average case. The regex patterns are compiled once at module load time, not on every classification call. These are small optimizations, but when you're classifying thousands of events per second they add up. + +The default return of `"info"` is deliberate. Most events in a production environment are routine. Classifying unknowns as high severity would create alert fatigue and train analysts to ignore alerts. + +### Step 6: Publishing to Redis Stream + +`app/core/streaming.py` handles the Redis Streams integration: + +```python +def publish_log_event(event: LogEvent) -> str | None: + """Publish a log event to the Redis log stream.""" + try: + data = { + "id": str(event.id), + "source_type": event.source_type, + "event_type": event.event_type, + "severity": event.severity, + "source_ip": event.source_ip or "", + "dest_ip": event.dest_ip or "", + "username": event.username or "", + "hostname": event.hostname or "", + "timestamp": event.timestamp.isoformat(), + } + stream_id = redis_client.xadd( + STREAM_LOGS, + data, + maxlen=STREAM_MAXLEN, + approximate=True, + ) + return stream_id + except Exception: + logger.exception("Failed to publish log event") + return None +``` + +Several things to note here: + +The `maxlen` parameter with `approximate=True` keeps the stream from growing without bound. Redis doesn't trim to exactly 10000 entries. Instead it trims in bulk when the stream exceeds the limit, which is more efficient than exact trimming. The stream might temporarily hold 10500 entries before Redis trims it back down. + +The `None` values get converted to empty strings because Redis Stream fields can't store `None`. Downstream consumers need to handle empty strings as missing values. + +The `try/except` with `return None` means a Redis failure doesn't crash the ingestion pipeline. The event is already persisted in MongoDB at this point. Losing the stream publish is acceptable. The event won't trigger real-time correlation or show up in the SSE feed until it's re-processed, but it won't be lost. + +### Step 7: Consumer Group Setup + +Before the correlation engine can read from the stream, consumer groups need to exist: + +```python +def ensure_consumer_group() -> None: + """Create consumer groups if they don't exist.""" + with contextlib.suppress(Exception): + redis_client.xgroup_create( + STREAM_LOGS, + CONSUMER_GROUP, + id="0", + mkstream=True, + ) + with contextlib.suppress(Exception): + redis_client.xgroup_create( + STREAM_ALERTS, + CONSUMER_GROUP, + id="0", + mkstream=True, + ) +``` + +The `contextlib.suppress(Exception)` handles the case where the consumer group already exists (Redis throws `BUSYGROUP`). This makes the function idempotent, safe to call on every startup. The `mkstream=True` parameter creates the stream itself if it doesn't exist. The `id="0"` means the consumer group starts reading from the beginning of the stream. + +## Correlation Engine + +This is the most complex part of the backend. The engine runs in a background thread, consuming events from Redis and evaluating them against correlation rules. + +### Engine Lifecycle + +`app/engine/correlation.py` defines the `CorrelationEngine` class: + +```python +class CorrelationEngine: + def __init__(self, app: Flask): + self.app = app + self.state = CorrelationState() + self._thread: threading.Thread | None = None + self._stop_event = threading.Event() + + def start(self) -> None: + self._thread = threading.Thread( + target=self._run, + daemon=True, + name="correlation-engine", + ) + self._thread.start() + + def stop(self) -> None: + self._stop_event.set() + if self._thread: + self._thread.join(timeout=5) +``` + +The engine starts as a daemon thread. Daemon threads are killed when the main process exits, so there's no risk of orphaned engine threads keeping the process alive. The `_stop_event` provides a clean shutdown path: the `_run()` loop checks it on every iteration. + +### The Main Loop + +```python +def _run(self) -> None: + with self.app.app_context(): + while not self._stop_event.is_set(): + try: + messages = redis_client.xreadgroup( + groupname=CONSUMER_GROUP, + consumername=CONSUMER_NAME, + streams={STREAM_LOGS: ">"}, + count=STREAM_READ_COUNT, + block=STREAM_BLOCK_MS, + ) + if messages: + for stream_name, entries in messages: + for entry_id, data in entries: + self._process_event(data, entry_id) + redis_client.xack( + STREAM_LOGS, + CONSUMER_GROUP, + entry_id, + ) + except Exception: + logger.exception("Correlation engine error") + time.sleep(1) +``` + +Let's break this down: + +`self.app.app_context()` pushes a Flask application context. Without this, the thread can't access MongoEngine (which is tied to Flask's app context) or the config. This is a common Flask gotcha with background threads. + +`xreadgroup` with `">"` reads only new, undelivered messages. The `block=STREAM_BLOCK_MS` (default 2000ms) makes this a blocking call. The thread sleeps for up to 2 seconds waiting for new events instead of busy-polling. When events arrive, it wakes up immediately. + +`count=STREAM_READ_COUNT` (default 10) limits how many events are read per call. This controls batch size. Reading 10 events at a time means the engine processes in small batches, which keeps per-event latency low while amortizing the Redis call overhead. + +After processing each event, `xack` acknowledges it. This tells Redis the event was successfully handled and doesn't need redelivery. If the engine crashes before acknowledging, Redis will redeliver the event on the next startup. This gives you at-least-once processing semantics. + +The `except Exception` with a 1-second sleep prevents a tight error loop. If Redis goes down, the engine logs the error and waits a second before retrying instead of hammering the dead connection. + +### Rule Loading and Caching + +```python +def _get_rules(self) -> list: + """Load enabled rules with caching.""" + now = time.time() + if now - self._rules_cache_time > RULES_CACHE_TTL: + self._rules_cache = list( + CorrelationRule.objects(enabled=True) + ) + self._rules_cache_time = now + return self._rules_cache +``` + +Rules are cached for 30 seconds (`RULES_CACHE_TTL`). Without caching, every event would trigger a MongoDB query for all enabled rules. With 100 events per second and 20 rules, that's 100 database queries per second just for rule loading. The cache reduces this to one query every 30 seconds. + +The tradeoff: when you create or modify a rule, it takes up to 30 seconds to take effect. For a SIEM, this is fine. Detection rules aren't time-critical to deploy. If you needed sub-second rule updates, you'd use a Redis pub/sub channel to notify the engine of changes. + +### Event Processing + +```python +def _process_event(self, data: dict, entry_id: str) -> None: + rules = self._get_rules() + for rule in rules: + try: + evaluator = EVALUATORS.get(rule.rule_type) + if evaluator and self._event_matches_rule(data, rule): + fired = evaluator(rule, data, self.state) + if fired: + self._create_alert(rule, data, fired) + except Exception: + logger.exception(f"Error evaluating rule {rule.id}") +``` + +For each event, the engine iterates over all enabled rules. First it checks whether the event is relevant to the rule (does the `event_type` match? does the `source_type` match?). Then it calls the appropriate evaluator based on rule type. + +The `EVALUATORS` dict maps rule types to functions: + +```python +EVALUATORS = { + "threshold": _evaluate_threshold, + "sequence": _evaluate_sequence, + "aggregation": _evaluate_aggregation, +} +``` + +This is the same registry pattern used in the normalizer. Adding a new rule type means writing an evaluator function and adding one entry to this dict. + +### Threshold Evaluation + +The threshold evaluator counts matching events per group key within a time window: + +```python +def _evaluate_threshold( + rule: CorrelationRule, + data: dict, + state: CorrelationState, +) -> list[str] | None: + conditions = rule.conditions + group_key = data.get(conditions.get("group_by", "source_ip"), "") + window = conditions.get("window_seconds", 300) + threshold = conditions.get("threshold", 10) + + # Add event to sliding window + state.add_event(rule.id, group_key, data, window) + + # Check if threshold exceeded + count = state.get_count(rule.id, group_key) + if count >= threshold: + if not state.in_cooldown(rule.id, group_key): + state.set_cooldown(rule.id, group_key) + return state.get_event_ids(rule.id, group_key) + + return None +``` + +The sliding window is maintained by `CorrelationState`. When we add an event, expired entries (older than `window_seconds`) are pruned. The count reflects only events within the current window. + +The cooldown check is critical. Without it, once the threshold is exceeded, every subsequent event would fire the rule again. A 10-minute brute force attack with threshold 10 and window 300 would generate hundreds of alerts without cooldown. With the default 300-second cooldown, it generates at most 2 alerts (one at the threshold, potentially another after the cooldown expires if the attack continues). + +### Sequence Evaluation + +Sequence rules are more complex. They track ordered sets of event types: + +```python +def _evaluate_sequence( + rule: CorrelationRule, + data: dict, + state: CorrelationState, +) -> list[str] | None: + conditions = rule.conditions + steps = conditions.get("steps", []) + group_key = data.get(conditions.get("group_by", "source_ip"), "") + window = conditions.get("window_seconds", 300) + + event_type = data.get("event_type", "") + + # Find which step this event matches + for i, step in enumerate(steps): + if event_type == step.get("event_type"): + state.add_sequence_event( + rule.id, group_key, i, data, window + ) + break + + # Check if all steps are satisfied in order + if state.sequence_complete(rule.id, group_key, steps): + if not state.in_cooldown(rule.id, group_key): + state.set_cooldown(rule.id, group_key) + return state.get_sequence_event_ids(rule.id, group_key) + + return None +``` + +A sequence rule with steps `[login_failure (count >= 5), login_success]` means: within the window, we need at least 5 `login_failure` events from the same group key, followed by at least one `login_success`. The `sequence_complete()` method checks both the ordering and the per-step count requirements. + +This models real attack patterns. Brute force followed by compromise isn't just "both happened." It's "failures happened first, then success." A sequence rule respects that ordering. + +### Aggregation Evaluation + +Aggregation rules count distinct values: + +```python +def _evaluate_aggregation( + rule: CorrelationRule, + data: dict, + state: CorrelationState, +) -> list[str] | None: + conditions = rule.conditions + group_key = data.get(conditions.get("group_by", "source_ip"), "") + agg_field = conditions.get("aggregation_field", "dest_ip") + window = conditions.get("window_seconds", 60) + threshold = conditions.get("threshold", 20) + + agg_value = data.get(agg_field, "") + state.add_aggregation_event( + rule.id, group_key, agg_value, data, window + ) + + distinct_count = state.get_distinct_count(rule.id, group_key) + if distinct_count >= threshold: + if not state.in_cooldown(rule.id, group_key): + state.set_cooldown(rule.id, group_key) + return state.get_aggregation_event_ids(rule.id, group_key) + + return None +``` + +The classic use case: port scanning detection. One source IP connects to 20+ distinct destination IPs in a minute. The aggregation state tracks which destination IPs have been seen (using a set for O(1) membership checks) and fires when the set exceeds the threshold. + +### Thread-Safe State + +`CorrelationState` uses threading locks because it's accessed from the correlation engine thread while the main thread might be reading state for debugging or testing: + +```python +class CorrelationState: + def __init__(self): + self._lock = threading.Lock() + self._windows: dict[str, dict[str, list]] = {} + self._cooldowns: dict[str, dict[str, float]] = {} + + def add_event(self, rule_id, group_key, data, window_seconds): + with self._lock: + key = f"{rule_id}:{group_key}" + now = time.time() + + if key not in self._windows: + self._windows[key] = [] + + # Prune expired + cutoff = now - window_seconds + self._windows[key] = [ + e for e in self._windows[key] + if e["timestamp"] > cutoff + ] + + # Add new + self._windows[key].append({ + "timestamp": now, + "event_id": data.get("id"), + "data": data, + }) +``` + +The lock is coarse-grained: one lock for the entire state object. This is simpler than per-key locking and plenty fast for single-threaded consumption. If you needed to scale to multiple correlation engine threads, you'd want finer-grained locking or a lock-free concurrent data structure. + +The sliding window prunes on every insertion. Old entries beyond `window_seconds` are filtered out. This keeps memory bounded. Without pruning, the state would grow linearly with event volume and never shrink. + +### Alert Creation + +When a rule fires, the engine creates an Alert document: + +```python +def _create_alert(self, rule, data, event_ids): + with self.app.app_context(): + alert = Alert.create_from_rule( + rule=rule, + matched_event_ids=event_ids, + trigger_event=data, + ) + + # Publish to alert stream + publish_alert(alert) +``` + +`Alert.create_from_rule()` is a class method on the Alert model that builds and saves the document, copying the rule's `mitre_tactic`, `mitre_technique`, severity, and description. The `matched_event_ids` field stores references to the LogEvent documents that triggered the rule. This lets analysts drill down from an alert to the raw events. + +After saving, the alert is published to the `siem:alerts` Redis Stream. The frontend's SSE connection picks this up and shows a notification in the browser. + +## Authentication System + +### Password Hashing + +`app/core/auth.py` implements the auth layer: + +```python +from pwdlib import PasswordHash +from pwdlib.hashers.argon2 import Argon2Hasher + +_hasher = PasswordHash((Argon2Hasher(),)) + +DUMMY_HASH = _hasher.hash("dummy-password-for-timing") + +def hash_password(password: str) -> str: + return _hasher.hash(password) + +def verify_password_timing_safe(password: str, hash: str | None) -> bool: + """Verify password with constant-time behavior.""" + if hash is None: + # User doesn't exist - still do the work + _hasher.verify(password, DUMMY_HASH) + return False + return _hasher.verify(password, hash) +``` + +The `DUMMY_HASH` is computed once at module load time. When `hash` is `None` (user doesn't exist), the function still performs a full Argon2id verification against this dummy. The result is thrown away, but the time cost is the same. This prevents username enumeration via timing side channels. + +Why `pwdlib` instead of `passlib`? `pwdlib` is a newer library with a cleaner API and better Argon2id defaults. The default parameters (memory cost, time cost, parallelism) match current OWASP recommendations without manual tuning. + +### JWT Tokens + +```python +def create_access_token(user: User) -> str: + config = get_settings() + now = datetime.utcnow() + payload = { + "sub": str(user.id), + "username": user.username, + "role": user.role, + "iat": now, + "exp": now + timedelta(hours=config.JWT_EXPIRY_HOURS), + } + return jwt.encode(payload, config.SECRET_KEY, algorithm="HS256") + +def decode_access_token(token: str) -> dict: + config = get_settings() + return jwt.decode(token, config.SECRET_KEY, algorithms=["HS256"]) +``` + +The JWT contains the user's ID, username, and role. This means the `@endpoint()` decorator can check authorization without a database query for most requests. The role is embedded in the token. + +The tradeoff: you can't revoke a JWT before it expires. If you change a user's role or deactivate their account, the old token still works until expiry. The code mitigates this by loading the user from MongoDB on every authenticated request (in the `@endpoint` decorator) and checking `is_active`. This means a deactivated user gets blocked on the next request, not just at token expiry. + +### The Endpoint Decorator + +`app/core/decorators/endpoint.py` is the auth enforcement point: + +```python +def endpoint(roles: str | tuple | None = None): + def decorator(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + token = _extract_token(request) + if token is None: + raise AuthenticationError("Missing token") + + payload = decode_access_token(token) + user = User.objects(id=payload["sub"]).first() + + if user is None or not user.is_active: + raise AuthenticationError("Invalid or inactive user") + + if roles and user.role not in ( + roles if isinstance(roles, tuple) else (roles,) + ): + raise ForbiddenError("Insufficient permissions") + + g.current_user = user + return fn(*args, **kwargs) + return wrapper + return decorator +``` + +`_extract_token()` pulls the JWT from the `Authorization: Bearer ` header. For SSE endpoints, it also checks query parameters because the browser's `EventSource` API doesn't support custom headers. This is noted in the architecture doc as a known tradeoff. + +The decorator loads the full user from MongoDB on every request. This seems expensive, but it's necessary. The JWT could be stale (role changed, account deactivated). The database check ensures the current state is used for authorization decisions. + +`g.current_user` stores the user on Flask's request-scoped `g` object. Any function downstream can access the authenticated user via `g.current_user` without passing it through parameters. + +## The Decorator Stack + +The project uses three composable decorators that form a pipeline: `@endpoint`, `@S`, and `@R`. Understanding how they compose is important. + +### @S (Schema Validation) + +`app/core/decorators/schema.py`: + +```python +def S(body: type[BaseModel] | None = None, + query: type[BaseModel] | None = None): + def decorator(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + if body: + try: + parsed = body.model_validate(request.get_json()) + except ValidationError as e: + raise AppValidationError(str(e)) + kwargs["body"] = parsed + + if query: + try: + parsed = query.model_validate(request.args.to_dict()) + except ValidationError as e: + raise AppValidationError(str(e)) + kwargs["query"] = parsed + + return fn(*args, **kwargs) + return wrapper + return decorator +``` + +`@S` intercepts the request before the route function runs. It parses the request body and/or query parameters against Pydantic schemas. Validated data is injected as keyword arguments (`body=`, `query=`). If validation fails, it raises an `AppValidationError` that the error handler converts to a 422 response. + +### @R (Response Serialization) + +`app/core/decorators/response.py`: + +```python +def R(status_code: int = 200): + def decorator(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + result = fn(*args, **kwargs) + if isinstance(result, Response): + return result + return jsonify(result), status_code + return wrapper + return decorator +``` + +`@R` wraps the return value in `jsonify()` and sets the status code. If the function returns a Flask `Response` directly (like for streaming), it passes through unchanged. + +### Composition Order + +When you see a route like: + +```python +@bp.post("/") +@endpoint(roles=ADMIN) +@S(body=RuleCreateSchema) +@R(status_code=201) +def create_rule(body: RuleCreateSchema): + return rule_ctrl.create(body) +``` + +The decorators execute from bottom to top (outermost first): +1. `@endpoint` checks auth and sets `g.current_user` +2. `@S` validates the request body +3. The route function runs +4. `@R` serializes the response + +If auth fails, schema validation never runs. If validation fails, the route function never runs. Each decorator either passes control to the next one or short-circuits with an error. + +## Streaming and SSE + +### SSE Generator + +`app/core/streaming.py` implements the Server-Sent Events generator: + +```python +def sse_log_stream(last_id: str = "$"): + """Generator that yields SSE events from the log stream.""" + while True: + try: + results = redis_client.xread( + streams={STREAM_LOGS: last_id}, + count=10, + block=STREAM_BLOCK_MS, + ) + if results: + for stream_name, entries in results: + for entry_id, data in entries: + last_id = entry_id + yield f"data: {json.dumps(data)}\n\n" + else: + # No events within block timeout - send keepalive + yield ": keepalive\n\n" + except GeneratorExit: + break + except Exception: + logger.exception("SSE stream error") + yield ": error\n\n" + time.sleep(1) +``` + +Key details: + +`xread` (not `xreadgroup`) is used for SSE. Consumer groups provide exactly-once delivery, which is appropriate for the correlation engine. But SSE clients are ephemeral browsers that come and go. If a browser disconnects and reconnects, it should get current events, not replayed old ones. `xread` with `$` starts from the latest event. + +The keepalive comment (`: keepalive\n\n`) is an SSE comment (starts with `:`). The browser ignores it, but it keeps the HTTP connection alive. Without it, proxies and load balancers would time out the connection. Nginx's default `proxy_read_timeout` is 60 seconds. The Nginx config for this project sets it to 3600 seconds for SSE endpoints, but keepalives are still needed as a safety net. + +`GeneratorExit` fires when the client disconnects. This is Python's way of telling the generator to clean up. Without catching it, the generator would log a spurious error on every client disconnect. + +### SSE Route + +```python +@bp.get("/stream") +@endpoint() +def stream_logs(): + token = request.args.get("token") # SSE can't use headers + return Response( + sse_log_stream(), + mimetype="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", # Nginx hint + }, + ) +``` + +The `X-Accel-Buffering: no` header tells Nginx to disable response buffering for this endpoint. Combined with `proxy_buffering off` in the Nginx config, this ensures events flow through to the browser immediately instead of being buffered. + +## Scenario Playback System + +### YAML Playbooks + +The project includes attack scenario playbooks in `app/scenarios/playbooks/`. Each YAML file describes a sequence of events that simulate a real attack: + +```yaml +# brute_force_lateral.yml +name: "Brute Force with Lateral Movement" +description: "SSH brute force followed by lateral movement" +mitre_techniques: + - T1110.001 + - T1021.004 +steps: + - delay: 0 + event: + source_type: auth + event_type: login_failure + source_ip: "203.0.113.50" + username: root + hostname: web-01 + auth_method: ssh_password + failure_reason: invalid_password + # ... 19 more login_failure events with varying usernames + - delay: 2 + event: + source_type: auth + event_type: login_success + source_ip: "203.0.113.50" + username: admin + hostname: web-01 + auth_method: ssh_password + - delay: 5 + event: + source_type: auth + event_type: login_success + source_ip: "10.0.1.10" # Internal IP now + username: admin + hostname: db-01 + auth_method: ssh_key +``` + +The `delay` field (in seconds) spaces out events realistically. A real brute force doesn't happen in zero time. The delays let the correlation engine's sliding windows work correctly during playback. + +### Playbook Parser + +`app/scenarios/playbook.py` loads and validates these YAML files: + +```python +def load_playbook(name: str) -> dict: + playbook_dir = Path(__file__).parent / "playbooks" + path = playbook_dir / f"{name}.yml" + + if not path.exists(): + raise NotFoundError(f"Playbook '{name}' not found") + + with open(path) as f: + playbook = yaml.safe_load(f) + + # Validate structure + if "steps" not in playbook: + raise ValidationError("Playbook missing 'steps'") + + return playbook +``` + +`yaml.safe_load` is used instead of `yaml.load`. This is a security consideration. `yaml.load` can execute arbitrary Python code embedded in YAML (via `!!python/object` tags). `safe_load` rejects those. In a security tool, this matters more than most places. + +### Threaded Runner + +`app/scenarios/runner.py` executes playbooks in background threads: + +```python +class ScenarioRunner: + _active_runs: dict[str, threading.Thread] = {} + + @classmethod + def start(cls, playbook_name: str, run_id: str, app: Flask) -> None: + thread = threading.Thread( + target=cls._execute, + args=(playbook_name, run_id, app), + daemon=True, + ) + cls._active_runs[run_id] = thread + thread.start() + + @classmethod + def _execute(cls, playbook_name, run_id, app): + with app.app_context(): + playbook = load_playbook(playbook_name) + run = ScenarioRun.objects(id=run_id).first() + + try: + run.update(status="running") + for step in playbook["steps"]: + if run.reload().status == "cancelled": + break + + delay = step.get("delay", 0) + if delay > 0: + time.sleep(delay) + + # Ingest the event through the normal pipeline + event_data = step["event"] + event_data["scenario_run_id"] = run_id + _ingest_scenario_event(event_data) + + status = "cancelled" if run.reload().status == "cancelled" else "completed" + run.update(status=status) + except Exception as e: + run.update(status="failed", error=str(e)) + finally: + cls._active_runs.pop(run_id, None) +``` + +Events injected by scenarios go through the same ingestion pipeline as real events. They get normalized, classified, persisted, published to the stream, and correlated. The only difference is the `scenario_run_id` field, which tags them for later cleanup and filtering. + +The `run.reload().status == "cancelled"` check on every step allows stopping a running scenario. When a user clicks "Cancel" in the UI, the status is updated in MongoDB, and the next loop iteration picks it up. + +The `_active_runs` dict tracks running threads. The `cleanup_orphaned_runs()` function called during startup marks any runs with status "running" as "failed" since they were interrupted by a process restart. + +## Frontend Integration + +### SSE Hook + +`src/api/hooks/useEventStream.ts` connects the browser to the SSE endpoints: + +```typescript +export function useEventStream( + endpoint: 'logs' | 'alerts', + enabled: boolean = true +) { + const { token } = useAuthStore(); + const addEvent = useStreamStore((s) => s.addEvent); + const addAlert = useStreamStore((s) => s.addAlert); + + useEffect(() => { + if (!enabled || !token) return; + + const url = `${API_BASE}/${endpoint}/stream?token=${token}`; + const source = new EventSource(url); + + source.onmessage = (event) => { + const data = JSON.parse(event.data); + if (endpoint === 'logs') { + addEvent(data); + } else { + addAlert(data); + } + }; + + source.onerror = () => { + source.close(); + // Reconnect after delay + setTimeout(() => { + // React will re-run the effect + }, 3000); + }; + + return () => source.close(); + }, [token, endpoint, enabled]); +} +``` + +The token is passed as a query parameter because `EventSource` doesn't support custom headers. This is a well known limitation of the SSE API. The token is still transmitted over HTTPS (assuming production deployment), so it's encrypted in transit. + +The reconnect logic is simple: close the connection and let React's `useEffect` cleanup/re-run cycle handle reconnection. In production you'd want exponential backoff, but for a learning project this is sufficient. + +### Zustand Stores + +`src/core/stores/stream.store.ts` manages real-time event state: + +```typescript +interface StreamState { + events: LogEvent[]; + alerts: Alert[]; + addEvent: (event: LogEvent) => void; + addAlert: (alert: Alert) => void; + clearEvents: () => void; +} + +export const useStreamStore = create((set) => ({ + events: [], + alerts: [], + addEvent: (event) => + set((state) => ({ + events: [...state.events, event].slice(-100), + })), + addAlert: (alert) => + set((state) => ({ + alerts: [...state.alerts, alert].slice(-50), + })), + clearEvents: () => set({ events: [], alerts: [] }), +})); +``` + +The `.slice(-100)` caps the in-memory event buffer at 100 entries. Without this, a sustained event stream would grow the browser's memory without bound. The UI shows the latest 100 events in real time, and users can query the full history via the search API. + +### Auth Store + +`src/core/stores/auth.store.ts` handles authentication state with persistence: + +```typescript +export const useAuthStore = create()( + persist( + (set) => ({ + token: null, + user: null, + login: (token, user) => set({ token, user }), + logout: () => set({ token: null, user: null }), + }), + { + name: 'siem-auth', + } + ) +); +``` + +Zustand's `persist` middleware saves the token and user to `localStorage`. This means refreshing the page doesn't log you out. The Axios interceptor in `src/core/lib/api.ts` reads the token from this store and attaches it to every API request: + +```typescript +api.interceptors.request.use((config) => { + const { token } = useAuthStore.getState(); + if (token) { + config.headers.Authorization = `Bearer ${token}`; + } + return config; +}); +``` + +### TanStack Query Hooks + +`src/api/hooks/` contains custom hooks for each API resource. The pattern is consistent: + +```typescript +// useAlerts.ts +export function useAlerts(params?: AlertQueryParams) { + return useQuery({ + queryKey: [QUERY_KEYS.ALERTS, params], + queryFn: () => api.get('/alerts', { params }).then((r) => r.data), + }); +} + +export function useUpdateAlertStatus() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: ({ id, status }: { id: string; status: string }) => + api.patch(`/alerts/${id}/status`, { status }), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: [QUERY_KEYS.ALERTS] }); + }, + }); +} +``` + +TanStack Query handles caching, deduplication, and background refetching. When an alert's status is updated, `invalidateQueries` forces a refresh of the alerts list. This keeps the UI consistent without manual state management. + +## Dashboard Aggregations + +The dashboard page shows summary visualizations: timeline, severity breakdown, and top sources. These are powered by MongoDB aggregation pipelines. + +### Timeline Aggregation + +`app/controllers/dashboard_ctrl.py`: + +```python +def get_timeline(hours: int = 24, bucket_minutes: int = 15): + cutoff = datetime.utcnow() - timedelta(hours=hours) + + pipeline = [ + {"$match": {"timestamp": {"$gte": cutoff}}}, + {"$group": { + "_id": { + "$dateTrunc": { + "date": "$timestamp", + "unit": "minute", + "binSize": bucket_minutes, + } + }, + "count": {"$sum": 1}, + "critical": { + "$sum": {"$cond": [ + {"$eq": ["$severity", "critical"]}, 1, 0 + ]} + }, + "high": { + "$sum": {"$cond": [ + {"$eq": ["$severity", "high"]}, 1, 0 + ]} + }, + }}, + {"$sort": {"_id": 1}}, + ] + + return list(LogEvent.objects.aggregate(pipeline)) +``` + +`$dateTrunc` buckets timestamps into 15-minute intervals. Within each bucket, the pipeline counts total events and breaks them down by severity. The frontend renders this as a stacked area chart. + +This query hits the `timestamp` index on `LogEvent`. Without that index, MongoDB would scan every document. With millions of events, that's the difference between milliseconds and minutes. + +### Pivot Queries + +The pivot API lets analysts investigate specific indicators: + +```python +def pivot(field: str, value: str, limit: int = 100): + """Find all events matching a specific field value.""" + valid_fields = { + "source_ip", "dest_ip", "username", + "hostname", "event_type", "source_type", + } + if field not in valid_fields: + raise ValidationError(f"Invalid pivot field: {field}") + + return LogEvent.objects(**{field: value}).order_by( + "-timestamp" + ).limit(limit) +``` + +This is the investigation workflow. An analyst sees an alert, notes the `source_ip`, and pivots to find every event from that IP. The whitelist of valid fields prevents arbitrary field queries that could be slow or exploitable. + +## Error Handling + +### Error Hierarchy + +`app/core/errors.py` defines a structured error hierarchy: + +```python +class AppError(Exception): + status_code = 500 + error_type = "AppError" + + def __init__(self, message: str): + self.message = message + super().__init__(message) + +class NotFoundError(AppError): + status_code = 404 + error_type = "NotFoundError" + +class ValidationError(AppError): + status_code = 422 + error_type = "ValidationError" + +class AuthenticationError(AppError): + status_code = 401 + error_type = "AuthenticationError" + +class ForbiddenError(AppError): + status_code = 403 + error_type = "ForbiddenError" + +class ConflictError(AppError): + status_code = 409 + error_type = "ConflictError" +``` + +A single Flask error handler catches all `AppError` subclasses: + +```python +def register_error_handlers(app: Flask): + @app.errorhandler(AppError) + def handle_app_error(error: AppError): + return jsonify({ + "error": error.error_type, + "message": error.message, + }), error.status_code +``` + +This gives you consistent JSON error responses across the entire API. No endpoint returns HTML errors. No endpoint returns different error formats. The frontend can rely on `response.data.error` and `response.data.message` being present for all error responses. + +### Frontend Error Handling + +`src/core/lib/api.ts` mirrors this on the client: + +```typescript +class ApiError extends Error { + constructor( + public status: number, + public error: string, + message: string + ) { + super(message); + } +} + +api.interceptors.response.use( + (response) => response, + (error) => { + if (error.response) { + const { status, data } = error.response; + throw new ApiError(status, data.error, data.message); + } + throw new ApiError(0, 'NetworkError', 'Connection failed'); + } +); +``` + +The Axios interceptor transforms HTTP errors into typed `ApiError` objects. React Query's global `onError` handler can then show toast notifications with the error message. A 401 triggers a logout. A 403 shows "Insufficient permissions." A 422 shows the validation error details. + +## Common Debugging Scenarios + +### "Events aren't showing up in real time" + +Check these in order: +1. Is the SSE connection established? Open browser dev tools, Network tab, filter by EventStream. You should see a long-lived request to `/api/v1/logs/stream`. +2. Is the Nginx proxy buffering disabled? Check `proxy_buffering off` in the Nginx config for SSE endpoints. +3. Is Redis running? `docker compose exec redis redis-cli ping` should return `PONG`. +4. Are events being published? `docker compose exec redis redis-cli XLEN siem:logs` shows the stream length. + +### "Correlation rules aren't firing" + +1. Is the correlation engine thread running? Check backend logs for "Correlation engine started." +2. Is the rule enabled? `CorrelationRule.objects(enabled=True)` in a Flask shell. +3. Does the event match the rule? The `_event_matches_rule()` function checks `event_type` and `source_type` against the rule's conditions. +4. Is the rule in cooldown? After firing once, it won't fire again for `CORRELATION_COOLDOWN_SECONDS` (default 300). +5. Has the rule cache expired? Rules are cached for 30 seconds. New rules take up to 30 seconds to be evaluated. + +### "Authentication errors on every request" + +1. Check the JWT expiry. Default is 24 hours (`JWT_EXPIRY_HOURS` in config). +2. Check that `SECRET_KEY` is consistent across restarts. If the key changes, all existing tokens become invalid. +3. Check the `Authorization` header format. It must be `Bearer `, not just the token. + +## What's Next + +Now that you understand how the code works, read [04-CHALLENGES.md](./04-CHALLENGES.md) for ideas on extending this project. The challenges range from adding new log source normalizers (beginner) to implementing Sigma rule support and SOAR playbooks (advanced). diff --git a/PROJECTS/intermediate/siem-dashboard/learn/04-CHALLENGES.md b/PROJECTS/intermediate/siem-dashboard/learn/04-CHALLENGES.md new file mode 100644 index 00000000..e8fa280e --- /dev/null +++ b/PROJECTS/intermediate/siem-dashboard/learn/04-CHALLENGES.md @@ -0,0 +1,1597 @@ +# 04-CHALLENGES.md + +# Extension Challenges + +You've built the base project. Now make it yours by extending it with new features. + +These challenges are ordered by difficulty. Start with the easier ones to build confidence, then tackle the harder ones when you want to dive deeper. + +## Easy Challenges + +### Challenge 1: Add CORS Misconfiguration Detection + +**What to build:** +A scanner that tests for overly permissive CORS (Cross-Origin Resource Sharing) policies. Check if the API accepts requests from any origin, reflects the Origin header, or allows credentials with wildcard origins. + +**Why it's useful:** +CORS misconfigurations let attackers steal data from authenticated users. If an API accepts `Origin: https://evil.com` and responds with `Access-Control-Allow-Origin: https://evil.com`, the attacker can make requests from their site and read the responses. This was how the PayPal information disclosure vulnerability (2020) worked. + +**What you'll learn:** +- HTTP header analysis and pattern matching +- Origin header manipulation techniques +- Understanding CORS security model +- How browsers enforce same-origin policy + +**Hints:** +- Look at `backend/scanners/auth_scanner.py` for header manipulation examples +- The scanner needs to make requests with different `Origin` headers +- Check response for `Access-Control-Allow-Origin`, `Access-Control-Allow-Credentials` +- Test cases: wildcard (`*`), reflected origin, `null` origin + +**Test it works:** +```python +# In scanners/cors_scanner.py +origins_to_test = [ + "https://evil.com", + "null", + "https://attacker.com", +] + +for origin in origins_to_test: + response = self.make_request("GET", "/", headers={"Origin": origin}) + + allow_origin = response.headers.get("Access-Control-Allow-Origin") + allow_creds = response.headers.get("Access-Control-Allow-Credentials") + + if allow_origin == origin or allow_origin == "*": + # Found CORS misconfiguration +``` + +Verify by scanning an endpoint with permissive CORS. Should detect `Access-Control-Allow-Origin: *` as vulnerable. + +### Challenge 2: Add Password Strength Reporting + +**What to build:** +Enhance registration to check password strength and provide feedback. Test against common password lists (rockyou.txt top 10000), check for patterns (keyboard walks like "qwerty"), and calculate entropy. + +**Why it's useful:** +Weak passwords are the #1 cause of account takeovers. The LinkedIn breach (2012) exposed 117 million passwords, many were "123456" and "password". Real-time feedback helps users create stronger passwords. + +**What you'll learn:** +- Password entropy calculation (bits of randomness) +- Pattern detection in strings +- Working with wordlists and datasets +- Balancing security with usability + +**Hints:** +- Modify `backend/schemas/user_schemas.py:26-38` (the password validator) +- Download common password lists from SecLists on GitHub +- Calculate entropy: `log2(character_space ^ length)` +- Check for sequential characters, repeated characters, dictionary words + +**Implementation approach:** + +1. **Add password checker utility** in `backend/core/security.py`: +```python +def check_password_strength(password: str) -> dict[str, Any]: + score = 0 + feedback = [] + + # Length check + if len(password) >= 12: + score += 2 + elif len(password) >= 8: + score += 1 + else: + feedback.append("Password should be at least 12 characters") + + # Character variety + if re.search(r"[A-Z]", password): + score += 1 + if re.search(r"[a-z]", password): + score += 1 + if re.search(r"[0-9]", password): + score += 1 + if re.search(r"[^A-Za-z0-9]", password): + score += 1 + + # Check against common passwords + common_passwords = load_common_passwords() # Load from file + if password.lower() in common_passwords: + score = 0 + feedback.append("This is a commonly used password") + + # Patterns + if re.search(r"(.)\1{2,}", password): # Repeated chars + score -= 1 + feedback.append("Avoid repeated characters") + + strength = "weak" if score < 3 else "medium" if score < 5 else "strong" + + return { + "strength": strength, + "score": score, + "feedback": feedback, + } +``` + +2. **Update registration endpoint** to return strength info (optional, informational only). + +**Test edge cases:** +- `password123` - Common pattern +- `P@ssw0rd` - Meets requirements but still weak +- `correct-horse-battery-staple` - Long passphrase (good) +- `aaaaAAAA1111!!!!` - Meets requirements but has patterns + +### Challenge 3: Add Response Time Monitoring Dashboard + +**What to build:** +Track and visualize scanner performance metrics. Store response times for each test, calculate percentiles (p50, p95, p99), and detect when targets are slowing down. + +**Why it's useful:** +Performance data helps tune scanner timeouts and detect issues. If the SQLi scanner's time-based detection shows high variance, you know the baseline isn't reliable. If all tests against a target suddenly take 10x longer, the target might be rate limiting you. + +**What you'll learn:** +- Metrics collection and aggregation +- Percentile calculation (not just averages) +- Time-series data visualization +- Performance baseline establishment + +**Hints:** +- `BaseScanner.make_request()` already tracks `request_time` at line 117 +- Store timing data in new `ScanMetrics` model +- Calculate percentiles with `numpy.percentile()` +- Frontend can use Chart.js or Recharts to visualize + +**Extra credit:** +Add alerting when p95 response time exceeds threshold. If average scan takes 30s but p95 is 120s, some scans are timing out frequently. + +## Intermediate Challenges + +### Challenge 4: Implement Stored XSS Detection + +**What to build:** +Extend XSS testing beyond reflected XSS. Submit payloads via POST, then retrieve the resource via GET to check if the payload persists. Test comment fields, user profiles, any data that gets stored and displayed. + +**Why it's useful:** +Stored XSS is more dangerous than reflected because it affects all users, not just the victim who clicks a malicious link. The MySpace Samy worm (2005) used stored XSS to infect over 1 million profiles in 20 hours. + +**Real world application:** +Any API with user-generated content needs stored XSS testing. Forums, comment systems, profile pages, file uploads with previews. + +**What you'll learn:** +- Multi-step testing workflows (submit then retrieve) +- Payload encoding variations (URL encoding, HTML entities, Unicode) +- Context-aware XSS detection (JavaScript context vs HTML context) +- False positive reduction in fuzzy matching + +**Implementation approach:** + +1. **Create stored XSS scanner** in `backend/scanners/stored_xss_scanner.py`: +```python +class StoredXSSScanner(BaseScanner): + def scan(self) -> TestResultCreate: + # Generate unique marker + marker = f"XSS_{uuid.uuid4().hex[:8]}" + payload = f"" + + # Step 1: Submit payload + submit_result = self._submit_payload(payload, marker) + if not submit_result["submitted"]: + return self._safe_result("Could not submit payload") + + # Step 2: Retrieve and check + retrieve_result = self._retrieve_and_check(marker) + if retrieve_result["vulnerable"]: + return self._vulnerable_result(payload, marker, retrieve_result) + + return self._safe_result("No stored XSS detected") + + def _submit_payload(self, payload: str, marker: str) -> dict[str, Any]: + # Try common endpoints + endpoints = ["/comments", "/api/posts", "/api/profile"] + + for endpoint in endpoints: + try: + response = self.make_request( + "POST", + endpoint, + json={"content": payload, "text": payload, "bio": payload} + ) + + if response.status_code in (200, 201): + return {"submitted": True, "endpoint": endpoint} + except Exception: + continue + + return {"submitted": False} + + def _retrieve_and_check(self, marker: str) -> dict[str, Any]: + # Retrieve content to see if payload persists + response = self.make_request("GET", "/") + + if marker in response.text: + # Check if it's encoded + if f"<script>" in response.text: + return {"vulnerable": False, "encoded": True} + + # Check if CSP would block it + csp = response.headers.get("Content-Security-Policy", "") + if "script-src 'none'" in csp or "script-src 'self'" in csp: + return {"vulnerable": False, "csp_protected": True} + + return {"vulnerable": True, "marker": marker} + + return {"vulnerable": False} +``` + +2. **Add cleanup** to remove test payloads after scanning (good citizenship). + +3. **Test edge cases:** +- Payload gets HTML encoded (safe) +- Payload stored but CSP prevents execution (still report as stored XSS) +- Payload appears in JSON response (context matters) + +**Gotchas:** +- Don't leave test payloads in production systems (always clean up) +- Some systems delay rendering (cache, async processing) so marker might not appear immediately +- Be careful with user attribution - don't associate test payloads with real users + +### Challenge 5: Add API Rate Limit Bypass Testing with Header Rotation + +**What to build:** +Extend rate limit bypass testing with more sophisticated techniques. Test User-Agent rotation, session ID manipulation, timestamp fuzzing, and Origin header variations. + +**Why this is challenging:** +Modern rate limiters use multiple signals (IP, user agent, session, fingerprint). You need to test combinations systematically. + +**What you'll learn:** +- Advanced rate limiting evasion techniques +- HTTP header manipulation at scale +- Statistical analysis of rate limit effectiveness +- Designing test matrices (combinatorial testing) + +**Implementation approach:** + +1. **Extend rate limit scanner** in `backend/scanners/rate_limit_scanner.py`: +```python +def _test_header_rotation_bypass(self) -> dict[str, Any]: + """ + Test if rotating headers bypasses rate limits + """ + user_agents = RateLimitBypassPayloads.USER_AGENT_ROTATION + + # Establish that rate limit exists + for _ in range(20): + response = self.make_request("GET", "/") + if response.status_code == 429: + break + else: + return {"bypass_successful": False, "reason": "No rate limit found"} + + # Try bypassing with User-Agent rotation + success_count = 0 + for i in range(20): + ua = user_agents[i % len(user_agents)] + + response = self.make_request( + "GET", "/", + headers={"User-Agent": ua} + ) + + if response.status_code != 429: + success_count += 1 + else: + break + + if success_count == 20: + return { + "bypass_successful": True, + "bypass_method": "User-Agent Rotation", + "requests_completed": success_count, + } + + return {"bypass_successful": False} +``` + +2. **Test combinations**: +```python +combinations = [ + {"User-Agent": ua, "X-Forwarded-For": ip}, + {"User-Agent": ua, "Origin": origin}, + # etc +] +``` + +3. **Add timing analysis** to detect soft limits (degraded but not blocked). + +**Resources:** +- Read "Bypassing Rate Limits" on PortSwigger Research blog +- Study Cloudflare's rate limiting documentation to understand what you're up against + +### Challenge 6: Implement XML External Entity (XXE) Detection + +**What to build:** +Test for XXE vulnerabilities in APIs that accept XML. Submit payloads with external entity references and check for file disclosure, SSRF, or denial of service. + +**Why this is hard:** +XXE requires understanding XML parsers, DTD syntax, and different attack vectors (file disclosure vs SSRF vs billion laughs). + +**What you'll learn:** +- XML parsing vulnerabilities +- Out-of-band data exfiltration techniques +- SSRF exploitation through XML +- Parser configuration security + +**Implementation:** + +Create `backend/scanners/xxe_scanner.py`: + +```python +class XXEScanner(BaseScanner): + def scan(self) -> TestResultCreate: + # Test file disclosure + file_disclosure = self._test_file_disclosure() + if file_disclosure["vulnerable"]: + return self._vulnerable_result( + "XXE file disclosure detected", + file_disclosure, + Severity.CRITICAL + ) + + # Test SSRF + ssrf_test = self._test_ssrf_xxe() + if ssrf_test["vulnerable"]: + return self._vulnerable_result( + "XXE SSRF detected", + ssrf_test, + Severity.HIGH + ) + + return self._safe_result() + + def _test_file_disclosure(self) -> dict[str, Any]: + # Payload to read /etc/passwd + payload = """ + + ]> + &xxe;""" + + response = self.make_request( + "POST", + "/", + data=payload, + headers={"Content-Type": "application/xml"} + ) + + # Check if file contents leaked + if "root:" in response.text or "bin/bash" in response.text: + return { + "vulnerable": True, + "payload": payload, + "leaked_data": response.text[:200] + } + + return {"vulnerable": False} + + def _test_ssrf_xxe(self) -> dict[str, Any]: + # Test if parser makes external requests + # Use Burp Collaborator or similar out-of-band detection + + collaborator_url = "http://burpcollaborator.net/unique-id" + + payload = f""" + + ]> + &xxe;""" + + response = self.make_request( + "POST", + "/", + data=payload, + headers={"Content-Type": "application/xml"} + ) + + # Check if request was made (need out-of-band detection) + # This is simplified - real implementation needs callback server + + return {"vulnerable": False, "note": "Manual verification required"} +``` + +**Success criteria:** +- Detects XXE in XML endpoints +- Tests multiple entity types (file, http, parameter entities) +- Handles different parser responses (error messages, timeout, data) +- Reports severity based on impact (file disclosure = CRITICAL, SSRF = HIGH) + +## Advanced Challenges + +### Challenge 7: Build a Scanner Plugin System + +**What to build:** +Create a plugin architecture that lets users write custom scanners without modifying core code. Scanners should be loadable from a `plugins/` directory, with automatic discovery and registration. + +**Why this is hard:** +Requires dynamic module loading, interface contracts, error isolation (broken plugin shouldn't crash scanner), and documentation for plugin developers. + +**What you'll learn:** +- Python module introspection and dynamic imports +- Abstract base classes and interface design +- Plugin architecture patterns +- Sandboxing and error isolation + +**Architecture changes needed:** + +``` +Current: +scanner_mapping = { + TestType.SQLI: SQLiScanner, # Hardcoded + ... +} + +New: +scanner_registry = ScannerRegistry() +scanner_registry.load_builtin_scanners() +scanner_registry.discover_plugins("plugins/") + +scanner_mapping = scanner_registry.get_all_scanners() +``` + +**Implementation steps:** + +1. **Define plugin interface** in `backend/scanners/plugin_interface.py`: +```python +from abc import ABC, abstractmethod +from typing import Any + +class ScannerPlugin(ABC): + """ + Base class for scanner plugins + + All plugins must inherit from this and implement required methods + """ + + @property + @abstractmethod + def name(self) -> str: + """Unique scanner name (e.g., 'custom_xxe')""" + pass + + @property + @abstractmethod + def version(self) -> str: + """Plugin version (semver: '1.0.0')""" + pass + + @property + @abstractmethod + def test_type(self) -> str: + """Test type identifier (must be unique)""" + pass + + @abstractmethod + def scan(self, target_url: str, **kwargs: Any) -> dict[str, Any]: + """ + Execute scan and return results + + Returns: + dict with keys: vulnerable, details, evidence, recommendations + """ + pass + + def validate(self) -> bool: + """ + Validate plugin configuration + Override this to add custom validation + """ + return True +``` + +2. **Create plugin loader** in `backend/scanners/plugin_loader.py`: +```python +import os +import importlib.util +from pathlib import Path +from typing import Type + +class PluginLoader: + def __init__(self, plugin_dir: str = "plugins"): + self.plugin_dir = Path(plugin_dir) + self.loaded_plugins: dict[str, Type[ScannerPlugin]] = {} + + def discover_plugins(self) -> list[Type[ScannerPlugin]]: + """ + Discover and load all plugins from plugin directory + """ + if not self.plugin_dir.exists(): + return [] + + plugins = [] + + for file in self.plugin_dir.glob("*.py"): + if file.stem.startswith("_"): + continue # Skip __init__.py, _template.py + + try: + plugin_class = self._load_plugin_from_file(file) + if plugin_class: + plugins.append(plugin_class) + except Exception as e: + print(f"Failed to load plugin {file}: {e}") + continue # Don't let broken plugins crash the scanner + + return plugins + + def _load_plugin_from_file(self, filepath: Path) -> Type[ScannerPlugin] | None: + """ + Dynamically load plugin class from Python file + """ + spec = importlib.util.spec_from_file_location( + f"plugins.{filepath.stem}", + filepath + ) + + if spec is None or spec.loader is None: + return None + + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + # Find ScannerPlugin subclass + for attr_name in dir(module): + attr = getattr(module, attr_name) + + if (isinstance(attr, type) and + issubclass(attr, ScannerPlugin) and + attr is not ScannerPlugin): + + # Instantiate to validate + instance = attr() + if instance.validate(): + return attr + + return None +``` + +3. **Create example plugin** in `plugins/example_scanner.py`: +```python +from scanners.plugin_interface import ScannerPlugin +from scanners.base_scanner import BaseScanner + +class ExampleScanner(ScannerPlugin, BaseScanner): + """ + Example scanner plugin + + Copy this to create your own scanners + """ + + @property + def name(self) -> str: + return "example_scanner" + + @property + def version(self) -> str: + return "1.0.0" + + @property + def test_type(self) -> str: + return "example" + + def scan(self, target_url: str, **kwargs) -> dict[str, Any]: + """ + Your scanner logic here + """ + self.target_url = target_url + + response = self.make_request("GET", "/") + + # Implement your detection logic + vulnerable = self._check_for_vulnerability(response) + + return { + "vulnerable": vulnerable, + "details": "Example vulnerability found" if vulnerable else "Safe", + "evidence": {"status_code": response.status_code}, + "recommendations": ["Fix the issue"] if vulnerable else [], + } + + def _check_for_vulnerability(self, response) -> bool: + # Your detection logic + return False +``` + +**Testing strategy:** +- Unit test plugin discovery (create test plugins in temp directory) +- Test error isolation (broken plugin doesn't crash scanner) +- Test plugin versioning (handle multiple versions of same plugin) + +**Known challenges:** + +1. **Plugin naming conflicts** + - Problem: Two plugins register same `test_type` + - Hint: Use namespacing or first-come-first-served with warnings + +2. **Plugin security** + - Problem: Malicious plugins could access database, file system + - Hint: Run plugins in restricted mode, limit imports, use subprocess isolation + +**Resources:** +- Read about Python's `importlib` module +- Study Flask's extension system for inspiration +- Look at pytest's plugin architecture + +### Challenge 8: Implement Blind SSRF Detection with Out-of-Band Channels + +**What to build:** +Detect Server-Side Request Forgery vulnerabilities even when responses don't leak data. Use DNS callbacks, HTTP callbacks, or timing attacks to confirm SSRF. + +**Why this is challenging:** +Blind SSRF doesn't show response data. You need infrastructure (callback server) to detect when target makes requests. + +**Estimated time:** +8-12 hours for basic implementation, 20+ hours for robust production version. + +**Prerequisites:** +You should have completed the SQLi and Auth challenges first because this builds on request timing analysis and header manipulation. + +**What you'll learn:** +- Out-of-band vulnerability detection techniques +- DNS exfiltration and callback mechanisms +- Time-based blind detection with statistical significance +- Building supporting infrastructure (callback servers) + +**Planning this feature:** + +Before you code, think through: +- How does the callback server work? (DNS? HTTP?) +- What if target has egress filtering? (Can't make outbound requests) +- How do you match callbacks to scans? (Unique identifiers) +- What's your false positive rate? (Network noise, CDN prefetching) + +**High level architecture:** + +``` +Scanner Callback Server Target API + | | | + |--SSRF Payload-------------->| | + | (http://callback.io/abc) | | + | | | + | |<---HTTP Request------------| + | | (GET /abc) | + | | | + |<--Confirmation---------------| | + | (Received request abc) | | +``` + +**Implementation phases:** + +**Phase 1: Build Callback Server** (3-4 hours) + +Create `backend/ssrf_callback_server.py`: + +```python +from fastapi import FastAPI, Request +from datetime import datetime, timedelta +import asyncio + +app = FastAPI() + +# Store received callbacks +callbacks_received: dict[str, dict] = {} + +@app.get("/callback/{identifier}") +async def handle_callback(identifier: str, request: Request): + """ + Receive SSRF callback + """ + callbacks_received[identifier] = { + "timestamp": datetime.utcnow(), + "headers": dict(request.headers), + "client_ip": request.client.host, + } + + return {"status": "received"} + +@app.get("/check/{identifier}") +async def check_callback(identifier: str): + """ + Check if callback was received + """ + callback = callbacks_received.get(identifier) + + if callback: + # Clean up old callback + del callbacks_received[identifier] + return {"received": True, "data": callback} + + return {"received": False} +``` + +Run this on a public server (DigitalOcean, AWS) with a domain pointing to it. + +**Phase 2: Implement SSRF Scanner** (3-4 hours) + +Create `backend/scanners/ssrf_scanner.py`: + +```python +import uuid +import time + +class SSRFScanner(BaseScanner): + def __init__(self, target_url: str, callback_server: str, **kwargs): + super().__init__(target_url, **kwargs) + self.callback_server = callback_server # http://callback.io + + def scan(self) -> TestResultCreate: + # Test URL parameters + url_param_test = self._test_url_parameters() + if url_param_test["vulnerable"]: + return self._vulnerable_result(url_param_test) + + # Test headers + header_test = self._test_headers() + if header_test["vulnerable"]: + return self._vulnerable_result(header_test) + + return self._safe_result() + + def _test_url_parameters(self) -> dict[str, Any]: + # Generate unique identifier + identifier = str(uuid.uuid4()) + + # Payloads to test + params = ["url", "callback", "webhook", "redirect", "link"] + + for param in params: + callback_url = f"{self.callback_server}/callback/{identifier}" + + # Submit SSRF payload + response = self.make_request( + "GET", + f"/?{param}={callback_url}" + ) + + # Wait for callback + time.sleep(5) + + # Check if callback was received + check_response = requests.get( + f"{self.callback_server}/check/{identifier}" + ) + + if check_response.json()["received"]: + return { + "vulnerable": True, + "parameter": param, + "callback_data": check_response.json()["data"], + } + + return {"vulnerable": False} + + def _test_headers(self) -> dict[str, Any]: + """ + Test headers like Referer, X-Forwarded-Host for SSRF + """ + identifier = str(uuid.uuid4()) + callback_url = f"{self.callback_server}/callback/{identifier}" + + headers_to_test = [ + "Referer", + "X-Forwarded-Host", + "X-Original-URL", + "Host", + ] + + for header in headers_to_test: + response = self.make_request( + "GET", + "/", + headers={header: callback_url} + ) + + time.sleep(5) + + check_response = requests.get( + f"{self.callback_server}/check/{identifier}" + ) + + if check_response.json()["received"]: + return { + "vulnerable": True, + "header": header, + "callback_data": check_response.json()["data"], + } + + return {"vulnerable": False} +``` + +**Phase 3: Add DNS Callback Alternative** (2-3 hours) + +Some environments block HTTP but allow DNS. Use DNS callbacks: + +```python +def _test_dns_callback(self) -> dict[str, Any]: + """ + Use DNS exfiltration for SSRF detection + """ + identifier = uuid.uuid4().hex[:8] + dns_domain = f"{identifier}.callback.io" # Your DNS server + + # Submit payload that triggers DNS lookup + payloads = [ + f"http://{dns_domain}/", + f"https://{dns_domain}/", + f"//{dns_domain}/", + ] + + for payload in payloads: + self.make_request("GET", f"/?url={payload}") + + time.sleep(5) + + # Check DNS logs for lookup + # (Requires DNS server that logs queries) + + return {"vulnerable": False} # Implement DNS checking +``` + +**Phase 4: Add Time-Based Detection** (3-4 hours) + +For fully blind SSRF (no callbacks possible), use timing: + +```python +def _test_timing_based_ssrf(self) -> dict[str, Any]: + """ + Detect SSRF via timing differences + + Internal IPs respond fast, external IPs are slower + """ + # Baseline with external IP (slow) + baseline_times = [] + for _ in range(5): + start = time.time() + self.make_request("GET", "/?url=http://example.com/") + baseline_times.append(time.time() - start) + + baseline_avg = statistics.mean(baseline_times) + + # Test with internal IPs (should be faster if SSRF exists) + internal_ips = [ + "http://127.0.0.1/", + "http://localhost/", + "http://192.168.1.1/", + ] + + for ip in internal_ips: + test_times = [] + for _ in range(5): + start = time.time() + self.make_request("GET", f"/?url={ip}") + test_times.append(time.time() - start) + + test_avg = statistics.mean(test_times) + + # If internal IP is significantly faster, SSRF likely exists + if test_avg < (baseline_avg * 0.5): + return { + "vulnerable": True, + "technique": "timing-based", + "baseline_time": baseline_avg, + "internal_ip_time": test_avg, + } + + return {"vulnerable": False} +``` + +**Testing the scanner:** + +Set up a vulnerable test API: +```python +# test_api.py +from flask import Flask, request +import requests + +app = Flask(__name__) + +@app.route("/") +def ssrf_vulnerable(): + url = request.args.get("url") + if url: + # Vulnerable - makes request to user-supplied URL + requests.get(url) + return "OK" + +if __name__ == "__main__": + app.run(port=5000) +``` + +Run scanner against it: +```bash +# Should detect SSRF and receive callback +python -m backend.scanners.ssrf_scanner http://localhost:5000 +``` + +**Success criteria:** +- [ ] Detects SSRF in URL parameters +- [ ] Detects SSRF in headers +- [ ] Uses DNS callbacks when HTTP blocked +- [ ] Falls back to timing-based detection +- [ ] Handles network delays gracefully +- [ ] Cleans up test callbacks + +### Challenge 9: Build a Full Vulnerability Report Generator + +**What to build:** +Generate professional PDF reports of scan results with executive summary, technical details, remediation steps, and CVSS scoring. + +**Estimated time:** +10-15 hours for complete implementation with styling and charts. + +**What you'll learn:** +- PDF generation with ReportLab or WeasyPrint +- CVSS scoring calculation +- Data visualization (matplotlib, plotly) +- Report templating and styling +- Professional documentation standards + +**Implementation phases:** + +**Phase 1: Report Data Structure** (2-3 hours) + +Create `backend/services/report_service.py`: + +```python +from dataclasses import dataclass +from typing import List +import matplotlib.pyplot as plt +from reportlab.lib.pagesizes import letter +from reportlab.pdfgen import canvas + +@dataclass +class VulnerabilitySummary: + critical: int + high: int + medium: int + low: int + info: int + + @property + def total(self) -> int: + return self.critical + self.high + self.medium + self.low + self.info + +class ReportGenerator: + def __init__(self, scan_id: int, db: Session): + self.scan = ScanRepository.get_by_id(db, scan_id) + self.db = db + + def generate_pdf(self, output_path: str) -> str: + """ + Generate comprehensive PDF report + """ + c = canvas.Canvas(output_path, pagesize=letter) + width, height = letter + + # Cover page + self._add_cover_page(c, width, height) + c.showPage() + + # Executive summary + self._add_executive_summary(c, width, height) + c.showPage() + + # Vulnerability details + self._add_vulnerability_details(c, width, height) + + c.save() + return output_path + + def _add_cover_page(self, c, width, height): + c.setFont("Helvetica-Bold", 24) + c.drawString(100, height - 100, "Security Scan Report") + + c.setFont("Helvetica", 14) + c.drawString(100, height - 150, f"Target: {self.scan.target_url}") + c.drawString(100, height - 180, f"Date: {self.scan.scan_date}") + + # Add severity chart + self._add_severity_chart(c, width, height - 400) + + def _add_severity_chart(self, c, x, y): + """ + Create pie chart of vulnerabilities by severity + """ + summary = self._calculate_summary() + + # Create matplotlib chart + fig, ax = plt.subplots(figsize=(4, 4)) + + labels = ["Critical", "High", "Medium", "Low", "Info"] + sizes = [summary.critical, summary.high, summary.medium, summary.low, summary.info] + colors = ["#dc2626", "#ea580c", "#f59e0b", "#3b82f6", "#6b7280"] + + ax.pie(sizes, labels=labels, colors=colors, autopct="%1.1f%%") + + # Save to temp file and embed in PDF + chart_path = "/tmp/severity_chart.png" + plt.savefig(chart_path) + plt.close() + + c.drawImage(chart_path, x, y, width=300, height=300) +``` + +**Phase 2: CVSS Scoring** (2-3 hours) + +Add CVSS score calculation: + +```python +class CVSSCalculator: + """ + Calculate CVSS v3.1 scores for vulnerabilities + """ + + def calculate_score( + self, + test_result: TestResult + ) -> dict[str, Any]: + """ + Calculate CVSS score based on test result + """ + # Base metrics + av = self._attack_vector(test_result.test_name) # Network + ac = self._attack_complexity(test_result) # Low/High + pr = self._privileges_required(test_result) # None/Low/High + ui = self._user_interaction(test_result) # None/Required + s = "U" # Scope: Unchanged + c = self._confidentiality_impact(test_result) # High/Low/None + i = self._integrity_impact(test_result) + a = self._availability_impact(test_result) + + # Calculate base score using CVSS formula + # (Simplified - full implementation in CVSS spec) + + impact = 1 - ((1 - c) * (1 - i) * (1 - a)) + exploitability = 8.22 * av * ac * pr * ui + + if impact <= 0: + base_score = 0 + else: + base_score = min( + (impact + exploitability) * 1.08, + 10.0 + ) + + return { + "score": round(base_score, 1), + "severity": self._score_to_severity(base_score), + "vector": f"CVSS:3.1/AV:{av}/AC:{ac}/PR:{pr}/UI:{ui}/S:{s}/C:{c}/I:{i}/A:{a}", + } +``` + +**Phase 3: Remediation Guide** (2-3 hours) + +Add detailed remediation for each vulnerability type: + +```python +REMEDIATION_GUIDES = { + TestType.SQLI: { + "title": "SQL Injection Remediation", + "steps": [ + "1. Use parameterized queries (prepared statements) for all database operations", + "2. Never concatenate user input into SQL strings", + "3. Use ORMs like SQLAlchemy that handle escaping automatically", + "4. Implement input validation to reject suspicious patterns", + "5. Use least privilege database accounts", + ], + "code_example": """ +# Before (Vulnerable) +query = f"SELECT * FROM users WHERE email = '{email}'" + +# After (Safe) +query = db.query(User).filter(User.email == email).first() + """, + "references": [ + "OWASP SQL Injection Prevention Cheat Sheet", + "CWE-89: Improper Neutralization of Special Elements", + ], + }, + # ... other test types +} +``` + +**Success criteria:** +- [ ] Generates professional-looking PDFs +- [ ] Includes executive summary (high-level findings) +- [ ] Shows vulnerability breakdown by severity +- [ ] Calculates CVSS scores +- [ ] Provides specific remediation steps +- [ ] Includes code examples +- [ ] Charts and visualizations + +## Mix and Match + +Combine features for bigger projects: + +**Project Idea 1: Complete API Security Platform** +- Combine Challenge 7 (plugin system) + Challenge 9 (report generation) + Challenge 8 (SSRF detection) +- Add web UI for scheduling scans +- Add email notifications when vulnerabilities found +- Result: Production-ready continuous API security testing + +**Project Idea 2: CI/CD Security Integration** +- Combine Challenge 4 (stored XSS) + Challenge 6 (XXE) + Challenge 2 (password strength) +- Build GitHub Action that runs scans on every commit +- Fail builds if critical vulnerabilities found +- Result: Security testing in development pipeline + +## Real World Integration Challenges + +### Integrate with Slack for Notifications + +**The goal:** +Send Slack messages when scans complete or vulnerabilities are found. + +**What you'll need:** +- Slack workspace with admin access +- Slack App with incoming webhook +- Understanding of Slack's Block Kit for rich messages + +**Implementation plan:** + +1. **Create Slack App** at https://api.slack.com/apps +2. **Enable Incoming Webhooks** and get webhook URL +3. **Add to config** in `.env`: +```bash +SLACK_WEBHOOK_URL=https://hooks.slack.com/services/YOUR/WEBHOOK/URL +``` + +4. **Create notification service** in `backend/services/notification_service.py`: +```python +import requests +from typing import List + +class SlackNotifier: + def __init__(self, webhook_url: str): + self.webhook_url = webhook_url + + def send_scan_complete(self, scan: Scan): + """ + Send notification when scan completes + """ + summary = self._calculate_summary(scan) + + blocks = [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "🔒 Security Scan Complete" + } + }, + { + "type": "section", + "fields": [ + {"type": "mrkdwn", "text": f"*Target:*\n{scan.target_url}"}, + {"type": "mrkdwn", "text": f"*Date:*\n{scan.scan_date}"}, + ] + }, + { + "type": "section", + "fields": [ + {"type": "mrkdwn", "text": f"*Critical:* {summary.critical}"}, + {"type": "mrkdwn", "text": f"*High:* {summary.high}"}, + {"type": "mrkdwn", "text": f"*Medium:* {summary.medium}"}, + ] + } + ] + + if summary.critical > 0 or summary.high > 0: + blocks.append({ + "type": "section", + "text": { + "type": "mrkdwn", + "text": "⚠️ *Critical vulnerabilities found!* Review immediately." + } + }) + + requests.post(self.webhook_url, json={"blocks": blocks}) +``` + +5. **Hook into scan service** at end of `ScanService.run_scan()`: +```python +# After saving results +if settings.SLACK_WEBHOOK_URL: + notifier = SlackNotifier(settings.SLACK_WEBHOOK_URL) + notifier.send_scan_complete(scan) +``` + +**Watch out for:** +- Rate limits on Slack webhooks (1 message per second) +- Message size limits (3000 characters for text blocks) +- Error handling (webhook URL might be invalid) + +### Deploy to Production (AWS/DigitalOcean) + +**The goal:** +Get this running in production on real infrastructure. + +**What you'll learn:** +- Docker deployment to cloud +- Environment variable management in production +- SSL/TLS certificate setup +- Database backups and maintenance + +**Steps:** + +1. **Provision server** (DigitalOcean Droplet, AWS EC2) + - Ubuntu 24.04 LTS + - 2GB RAM minimum (4GB recommended) + - Open ports: 80 (HTTP), 443 (HTTPS), 22 (SSH) + +2. **Install Docker** on server: +```bash +ssh root@your-server-ip + +curl -fsSL https://get.docker.com -o get-docker.sh +sh get-docker.sh + +apt install docker-compose-plugin +``` + +3. **Clone project** and configure: +```bash +git clone https://github.com/yourusername/api-security-scanner.git +cd api-security-scanner + +cp .env.example .env +nano .env # Edit production settings +``` + +4. **Set production environment variables**: +```bash +# .env for production +SECRET_KEY=$(openssl rand -hex 32) +DEBUG=false +DATABASE_URL=postgresql://... +CORS_ORIGINS=https://yourdomain.com +``` + +5. **Set up SSL with Let's Encrypt**: +```bash +# Install certbot +apt install certbot python3-certbot-nginx + +# Get certificate +certbot --nginx -d yourdomain.com +``` + +6. **Update nginx config** in `conf/nginx/prod.nginx` to use SSL: +```nginx +server { + listen 443 ssl http2; + server_name yourdomain.com; + + ssl_certificate /etc/letsencrypt/live/yourdomain.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/yourdomain.com/privkey.pem; + + # ... rest of config +} +``` + +7. **Deploy**: +```bash +docker compose -f compose.yml up -d --build +``` + +**Production checklist:** +- [ ] Changed `SECRET_KEY` to random value +- [ ] Set `DEBUG=false` +- [ ] Configured real database (not localhost) +- [ ] Set up SSL certificates +- [ ] Configured firewall (ufw allow 80,443,22) +- [ ] Set up backups (database dumps to S3/Spaces) +- [ ] Configure monitoring (Prometheus, Grafana, or cloud provider) +- [ ] Set up logging (centralized logs, log rotation) + +## Performance Challenges + +### Challenge: Handle 1000 concurrent scans + +**The goal:** +Scale the system to handle 1000 scans running simultaneously without crashing or slowing down. + +**Current bottleneck:** +Scans run synchronously in request handler. Database connections limited to 5 by default. Memory usage grows linearly with concurrent scans. + +**Optimization approaches:** + +**Approach 1: Task Queue with Celery** +- How: Move scans to background workers with Celery + Redis +- Gain: Non-blocking API, horizontal scaling, retry logic +- Tradeoff: Added complexity, need Redis infrastructure + +Implementation: +```bash +# Install Celery +pip install celery redis + +# backend/celery_app.py +from celery import Celery + +celery = Celery( + "scanner", + broker="redis://localhost:6379/0", + backend="redis://localhost:6379/0" +) + +@celery.task +def run_scan_async(scan_id: int): + """Run scan in background""" + db = SessionLocal() + scan = ScanRepository.get_by_id(db, scan_id) + # ... execute scan +``` + +**Approach 2: Increase Database Connections** +- How: Configure connection pooling in SQLAlchemy +- Gain: More concurrent database operations +- Tradeoff: Higher memory usage, Postgres connection limits + +```python +# core/database.py +engine = create_engine( + settings.DATABASE_URL, + pool_size=20, # Increased from 5 + max_overflow=40, # Up to 60 total connections + pool_pre_ping=True, +) +``` + +**Benchmark it:** +```bash +# Load testing with Apache Bench +ab -n 1000 -c 100 http://localhost:8000/scans/ \ + -H "Authorization: Bearer TOKEN" \ + -p scan_request.json +``` + +Target metrics: +- Throughput: 50+ requests/second +- Latency p95: <2 seconds (for scan creation, not execution) +- Error rate: <1% + +### Challenge: Reduce Scanner Memory Usage + +**The goal:** +Cut memory usage by 50% when running 100 concurrent scans. + +**Profile first:** +```python +# Add memory profiling +import tracemalloc + +tracemalloc.start() + +# Run scan +scanner.scan() + +current, peak = tracemalloc.get_traced_memory() +print(f"Current: {current / 1024 / 1024:.2f} MB") +print(f"Peak: {peak / 1024 / 1024:.2f} MB") + +tracemalloc.stop() +``` + +**Common optimization areas:** +- Store response bodies in memory (large responses eat RAM) +- Session objects not being cleaned up +- Evidence JSON storing entire responses + +Fix: +```python +# Instead of storing full response +evidence = {"response": response.text} # 100KB+ + +# Store summary +evidence = { + "status_code": response.status_code, + "length": len(response.text), + "excerpt": response.text[:500], # Just first 500 chars +} +``` + +## Security Challenges + +### Challenge: Add Webhook Signature Verification + +**What to implement:** +When sending scan results to webhooks, sign the payload so receivers can verify it's from your scanner. + +**Threat model:** +This protects against: +- Attacker sending fake scan results to webhook +- Man-in-the-middle tampering with webhook data + +**Implementation:** + +```python +import hmac +import hashlib + +class WebhookSigner: + def __init__(self, secret: str): + self.secret = secret.encode() + + def sign(self, payload: str) -> str: + """ + Create HMAC-SHA256 signature of payload + """ + signature = hmac.new( + self.secret, + payload.encode(), + hashlib.sha256 + ).hexdigest() + + return signature + + def verify(self, payload: str, signature: str) -> bool: + """ + Verify signature is valid + """ + expected = self.sign(payload) + return hmac.compare_digest(expected, signature) +``` + +Usage: +```python +# When sending webhook +signer = WebhookSigner(settings.WEBHOOK_SECRET) +payload_json = json.dumps(scan_data) +signature = signer.sign(payload_json) + +requests.post( + webhook_url, + json=scan_data, + headers={"X-Signature": signature} +) + +# Receiver verifies +received_signature = request.headers["X-Signature"] +if not signer.verify(payload_json, received_signature): + raise HTTPException(401, "Invalid signature") +``` + +### Challenge: Pass OWASP Top 10 Compliance + +**The goal:** +Make this project compliant with OWASP Top 10 2021. + +**Current gaps:** +- A01:2021-Broken Access Control: IDOR checks implemented ✓ +- A02:2021-Cryptographic Failures: Bcrypt for passwords ✓, but no HTTPS enforcement +- A03:2021-Injection: SQL injection detection ✓, but could add command injection +- A07:2021-Identification and Authentication Failures: JWT validation ✓ +- A09:2021-Security Logging and Monitoring Failures: No audit logging ❌ + +**Remediation:** + +Add audit logging: +```python +# models/AuditLog.py +class AuditLog(BaseModel): + __tablename__ = "audit_logs" + + user_id = Column(Integer, ForeignKey("users.id")) + action = Column(String(100)) # "scan_created", "login_success" + ip_address = Column(String(45)) # Support IPv6 + user_agent = Column(String(255)) + details = Column(JSON) + +# Log every important action +def log_action(user_id: int, action: str, request: Request, **details): + log = AuditLog( + user_id=user_id, + action=action, + ip_address=request.client.host, + user_agent=request.headers.get("User-Agent"), + details=details, + ) + db.add(log) + db.commit() +``` + +## Contribution Ideas + +Finished a challenge? Share it back: + +1. **Fork the repo** at github.com/yourusername/api-security-scanner +2. **Implement your extension** in a feature branch (`git checkout -b feature/xxe-scanner`) +3. **Document it** - Add to learn folder, update README +4. **Submit a PR** with: + - Implementation code + - Unit tests (minimum 80% coverage) + - Integration tests + - Documentation (docstrings, README updates) + - Example usage + +Good extensions might get merged into the main project. + +## Challenge Yourself Further + +### Build Something New + +Use the concepts you learned here to build: + +- **GraphQL security scanner** - Test for introspection leaks, query depth limits, batching attacks +- **WebSocket security tester** - Test for injection, authentication, rate limiting in WebSocket connections +- **Cloud API scanner** - Test AWS, Azure, GCP APIs for misconfiguration (public S3 buckets, open databases) + +### Study Real Implementations + +Compare your implementation to production tools: + +- **Burp Suite** - Study how their active scanner detects SQLi (multiple techniques, adaptive testing) +- **OWASP ZAP** - Look at their scanner plugin architecture +- **Nuclei** - Check out their YAML-based template system for custom checks + +Read their code (many are open source), understand their tradeoffs, steal their good ideas. + +### Write About It + +Document your extension: + +- Blog post explaining what you built and why ("Adding XXE Detection to an API Scanner") +- Tutorial for others to follow ("How to Build a CORS Misconfiguration Scanner") +- Comparison with alternative approaches ("Callback-based SSRF Detection vs Timing-based") + +Teaching others is the best way to verify you understand it. + +## Getting Help + +Stuck on a challenge? + +1. **Debug systematically** + - What did you expect to happen? + - What actually happened? + - What's the smallest test case that reproduces it? + - What have you already tried? + +2. **Read the existing code** + - SQLi scanner does time-based detection (similar to SSRF timing) + - Auth scanner does multi-step testing (similar to stored XSS) + - Base scanner has retry logic you can reuse + +3. **Search for similar problems** + - Stack Overflow with tags: python, fastapi, security-testing + - GitHub issues in similar projects (ZAP, Nuclei, SQLMap) + - InfoSec forums like /r/netsec, /r/AskNetsec + +4. **Ask for help with context** + - Post in project discussions or issues + - Include: what you're trying to build, what you tried, what happened, what you expected + - Provide code snippets and error messages + - Don't just paste error messages without explanation + +## Challenge Completion Tracker + +Track your progress: + +- [ ] Easy Challenge 1: CORS Detection +- [ ] Easy Challenge 2: Password Strength +- [ ] Easy Challenge 3: Response Time Monitoring +- [ ] Intermediate Challenge 4: Stored XSS +- [ ] Intermediate Challenge 5: Advanced Rate Limit Bypass +- [ ] Intermediate Challenge 6: XXE Detection +- [ ] Advanced Challenge 7: Plugin System +- [ ] Advanced Challenge 8: Blind SSRF +- [ ] Expert Challenge 9: Report Generation + +**Bonus challenges:** +- [ ] Slack Integration +- [ ] Production Deployment +- [ ] Performance: 1000 Concurrent Scans +- [ ] Security: Webhook Signatures +- [ ] Compliance: OWASP Top 10 + +Completed all of them? You've mastered API security testing. Time to build something new, contribute to open source security tools, or apply these skills professionally in penetration testing or security engineering roles.