diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/health.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/health.py index d4085edd..83720b88 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/api/health.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/health.py @@ -5,8 +5,10 @@ health.py Health and readiness probe endpoints for container orchestration -GET /health returns liveness status with uptime_seconds -and pipeline_running flag. GET /ready checks database +GET /health returns liveness status with uptime_seconds, +pipeline_running flag, and per-stage pipeline_stats +counters (parsed/enriched/scored/dispatched with error +counts). GET /ready checks database connectivity (SELECT 1) and Redis ping, reports models_loaded status, and returns 503 if any dependency is down. Both endpoints read from app.state set during @@ -34,10 +36,12 @@ async def health(request: Request) -> dict[str, object]: Liveness probe — returns 200 if the process is alive. """ uptime = time.monotonic() - request.app.state.startup_time + pipeline = getattr(request.app.state, "pipeline", None) return { "status": "healthy", "uptime_seconds": round(uptime, 2), "pipeline_running": request.app.state.pipeline_running, + "pipeline_stats": pipeline.stats if pipeline else {}, } diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py index 4730f207..6f2d9915 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py @@ -6,14 +6,16 @@ ML model status and retraining endpoints GET /models/status returns models_loaded flag, detection _mode (hybrid or rules), and active model metadata from -the database. POST /models/retrain dispatches a +the database. POST /models/retrain acquires _retrain_lock +(returning 409 if already running), dispatches a background retraining job that loads stored ThreatEvents, labels them using review_label or score thresholds (SCORE_ATTACK_THRESHOLD 0.5, SCORE_NORMAL_CEILING 0.3), supplements with synthetic data if below MIN_TRAINING_ SAMPLES (200), runs TrainingOrchestrator, and writes model metadata. _fallback_synthetic spawns a subprocess -CLI train command when no real events exist +CLI train command with lifecycle tracking via +_synthetic_process Connects to: config.py - settings.model_dir, ensemble @@ -25,10 +27,13 @@ Connects to: cli/main - _write_metadata """ +import asyncio import logging +import subprocess import uuid -from fastapi import APIRouter, BackgroundTasks, Request +from fastapi import APIRouter, BackgroundTasks, Request, Response +from fastapi.responses import JSONResponse from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -46,6 +51,9 @@ MIN_TRAINING_SAMPLES = 200 SYNTHETIC_SUPPLEMENT_NORMAL = 500 SYNTHETIC_SUPPLEMENT_ATTACK = 250 +_retrain_lock = asyncio.Lock() +_synthetic_process: subprocess.Popen[bytes] | None = None + @router.get("/status") async def model_status(request: Request) -> dict[str, object]: @@ -68,15 +76,21 @@ async def model_status(request: Request) -> dict[str, object]: } -@router.post("/retrain", status_code=202) +@router.post("/retrain", status_code=202, response_model=None) async def retrain( request: Request, background_tasks: BackgroundTasks, -) -> dict[str, object]: +) -> dict[str, object] | Response: """ Dispatch a model retraining job using real stored threat events supplemented with synthetic data """ + if _retrain_lock.locked(): + return JSONResponse( + status_code=409, + content={"status": "conflict", "job_id": ""}, + ) + session_factory = getattr(request.app.state, "session_factory", None) if session_factory is None: return {"status": "error", "job_id": ""} @@ -99,7 +113,6 @@ async def _retrain_from_db( supplement with synthetic data if needed, and run the full training pipeline """ - import asyncio import dataclasses from pathlib import Path @@ -107,109 +120,117 @@ async def _retrain_from_db( from ml.orchestrator import TrainingOrchestrator - logger.info("Retrain job %s: loading stored events", job_id) + async with _retrain_lock: + logger.info("Retrain job %s: loading stored events", job_id) - async with session_factory() as session: - count = (await session.execute( - select(func.count()).select_from(ThreatEvent) - )).scalar_one() + async with session_factory() as session: + count = (await session.execute( + select(func.count()).select_from(ThreatEvent) + )).scalar_one() - if count == 0: - logger.warning( - "Retrain job %s: no stored events, using synthetic only", + if count == 0: + logger.warning( + "Retrain job %s: no stored events, " + "using synthetic only", + job_id, + ) + _fallback_synthetic(job_id) + return + + rows = (await session.execute( + select(ThreatEvent) + )).scalars().all() + + vectors: list[list[float]] = [] + labels: list[int] = [] + + for event in rows: + if not event.feature_vector: + continue + + if event.reviewed and event.review_label: + label = ( + 1 if event.review_label == "true_positive" + else 0 + ) + elif event.threat_score >= SCORE_ATTACK_THRESHOLD: + label = 1 + elif event.threat_score < SCORE_NORMAL_CEILING: + label = 0 + else: + continue + + vectors.append(event.feature_vector) + labels.append(label) + + logger.info( + "Retrain job %s: %d usable events from DB " + "(normal=%d, attack=%d)", + job_id, + len(vectors), + labels.count(0), + labels.count(1), + ) + + from ml.synthetic import generate_mixed_dataset + + if len(vectors) < MIN_TRAINING_SAMPLES: + syn_X, syn_y = generate_mixed_dataset( + SYNTHETIC_SUPPLEMENT_NORMAL, + SYNTHETIC_SUPPLEMENT_ATTACK, + ) + X = np.concatenate([ + np.array(vectors, dtype=np.float32), + syn_X, + ]) if vectors else syn_X + y = np.concatenate([ + np.array(labels, dtype=np.int32), + syn_y, + ]) if labels else syn_y + logger.info( + "Retrain job %s: supplemented with " + "%d synthetic samples", + job_id, + len(syn_X), + ) + else: + X = np.array(vectors, dtype=np.float32) + y = np.array(labels, dtype=np.int32) + + output_dir = Path(settings.model_dir) + loop = asyncio.get_running_loop() + result = await loop.run_in_executor( + None, + lambda: TrainingOrchestrator( + output_dir=output_dir, + ).run(X, y), + ) + + logger.info( + "Retrain job %s complete: passed_gates=%s", + job_id, + result.passed_gates, + ) + + try: + from cli.main import _write_metadata + + metrics: dict[str, object] = ( + dataclasses.asdict(result.ensemble_metrics) + if result.ensemble_metrics else {} + ) + await _write_metadata( + output_dir, + len(X), + metrics, + result.mlflow_run_id, + result.ae_metrics.get("ae_threshold"), + ) + except Exception: + logger.exception( + "Retrain job %s: failed to write metadata", job_id, ) - _fallback_synthetic(job_id) - return - - rows = (await session.execute( - select(ThreatEvent) - )).scalars().all() - - vectors: list[list[float]] = [] - labels: list[int] = [] - - for event in rows: - if not event.feature_vector: - continue - - if event.reviewed and event.review_label: - label = 1 if event.review_label == "true_positive" else 0 - elif event.threat_score >= SCORE_ATTACK_THRESHOLD: - label = 1 - elif event.threat_score < SCORE_NORMAL_CEILING: - label = 0 - else: - continue - - vectors.append(event.feature_vector) - labels.append(label) - - logger.info( - "Retrain job %s: %d usable events from DB " - "(normal=%d, attack=%d)", - job_id, - len(vectors), - labels.count(0), - labels.count(1), - ) - - from ml.synthetic import generate_mixed_dataset - - if len(vectors) < MIN_TRAINING_SAMPLES: - syn_X, syn_y = generate_mixed_dataset( - SYNTHETIC_SUPPLEMENT_NORMAL, - SYNTHETIC_SUPPLEMENT_ATTACK, - ) - X = np.concatenate([ - np.array(vectors, dtype=np.float32), - syn_X, - ]) if vectors else syn_X - y = np.concatenate([ - np.array(labels, dtype=np.int32), - syn_y, - ]) if labels else syn_y - logger.info( - "Retrain job %s: supplemented with %d synthetic samples", - job_id, - len(syn_X), - ) - else: - X = np.array(vectors, dtype=np.float32) - y = np.array(labels, dtype=np.int32) - - output_dir = Path(settings.model_dir) - loop = asyncio.get_running_loop() - result = await loop.run_in_executor( - None, - lambda: TrainingOrchestrator(output_dir=output_dir).run(X, y), - ) - - logger.info( - "Retrain job %s complete: passed_gates=%s", - job_id, - result.passed_gates, - ) - - try: - from cli.main import _write_metadata - - metrics: dict[str, object] = ( - dataclasses.asdict(result.ensemble_metrics) - if result.ensemble_metrics else {} - ) - await _write_metadata( - output_dir, - len(X), - metrics, - result.mlflow_run_id, - result.ae_metrics.get("ae_threshold"), - ) - except Exception: - logger.exception( - "Retrain job %s: failed to write metadata", - job_id, - ) def _fallback_synthetic(job_id: str) -> None: @@ -217,11 +238,30 @@ def _fallback_synthetic(job_id: str) -> None: Run training with synthetic data only when no real events exist """ - import subprocess + global _synthetic_process # noqa: PLW0603 import sys - logger.info("Retrain job %s: falling back to synthetic training", job_id) - subprocess.Popen( + if _synthetic_process is not None: + if _synthetic_process.poll() is None: + logger.info( + "Retrain job %s: synthetic training already " + "running (pid=%d)", + job_id, + _synthetic_process.pid, + ) + return + rc = _synthetic_process.returncode + if rc != 0: + logger.warning( + "Previous synthetic training exited with %d", + rc, + ) + + logger.info( + "Retrain job %s: falling back to synthetic training", + job_id, + ) + _synthetic_process = subprocess.Popen( [ sys.executable, "-m", diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/config.py b/PROJECTS/advanced/ai-threat-detection/backend/app/config.py index 93e4c49d..f3e1cb46 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/config.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/config.py @@ -13,8 +13,9 @@ database path, nginx log path, pipeline queue sizes settings (size 32, timeout 50ms), and ML configuration (model_dir, detection_mode, ensemble weights for autoencoder/random-forest/isolation-forest at 0.40/0.40 -/0.20, ae_threshold_percentile 99.5, MLflow tracking -URI). Exports a module-level singleton settings instance +/0.20 with model_validator enforcing sum-to-1.0, +ae_threshold_percentile 99.5, MLflow tracking URI). +Exports a module-level singleton settings instance Connects to: factory.py - consumed in lifespan and create_app @@ -24,6 +25,9 @@ Connects to: core/enrichment/ - geoip_db_path """ +from typing import Self + +from pydantic import model_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -70,5 +74,21 @@ class Settings(BaseSettings): ae_threshold_percentile: float = 99.5 mlflow_tracking_uri: str = "file:./mlruns" + @model_validator(mode="after") + def _check_ensemble_weights(self) -> Self: + """ + Validate that ensemble weights sum to 1.0 + """ + total = ( + self.ensemble_weight_ae + + self.ensemble_weight_rf + + self.ensemble_weight_if + ) + if abs(total - 1.0) > 1e-6: + raise ValueError( + f"Ensemble weights must sum to 1.0, got {total:.6f}" + ) + return self + settings = Settings() diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py index c2c67351..98bdc57c 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py @@ -104,6 +104,9 @@ class AlertDispatcher: request_path=scored.entry.path, threat_score=scored.final_score, severity=severity, - component_scores=scored.rule_result.component_scores, + component_scores={ + **scored.rule_result.component_scores, + **(scored.ml_scores or {}), + }, ) await self._redis.publish(ALERTS_CHANNEL, alert.model_dump_json()) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/inference.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/inference.py index 2111795b..5488b6cd 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/inference.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/inference.py @@ -31,6 +31,8 @@ from typing import Any import numpy as np +from app.core.features.mappings import FEATURE_ORDER + try: import onnxruntime as ort except ImportError: @@ -88,6 +90,18 @@ class InferenceEngine: self._if_session = ort.InferenceSession(str(if_path), opts) scaler_data = json.loads(scaler_path.read_text()) + + stored_names = scaler_data.get("feature_names") + if ( + stored_names is not None + and stored_names != list(FEATURE_ORDER) + ): + logger.error( + "Feature ordering mismatch in %s", + scaler_path, + ) + return + self._scaler_center = np.array(scaler_data["center"], dtype=np.float32) self._scaler_scale = np.array(scaler_data["scale"], diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/rules.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/rules.py index 9a43184b..41a65844 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/rules.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/rules.py @@ -5,10 +5,11 @@ rules.py Cold-start rule-based detection engine inspired by ModSecurity Core Rule Set -RuleEngine.score_request evaluates requests against 7 +RuleEngine.score_request evaluates requests against 9 regex-based _PatternRules (LOG4SHELL 0.95, COMMAND_ INJECTION 0.90, SQL_INJECTION 0.85, XSS 0.80, FILE_ -INCLUSION 0.75, SSRF 0.70, PATH_TRAVERSAL 0.60), +INCLUSION 0.75, SSRF 0.70, CRLF_INJECTION 0.65, +PATH_TRAVERSAL 0.60, OPEN_REDIRECT 0.55), double-encoding detection (0.40), scanner user-agent signature matching (0.35), and 2 _ThresholdRules (RATE_ANOMALY >100 req/min 0.30, HIGH_ERROR_RATE >50% @@ -20,7 +21,8 @@ and component_scores Connects to: core/features/ patterns - compiled regex patterns (SQLI, - XSS, LOG4SHELL, etc.) + XSS, LOG4SHELL, CRLF_INJECTION, + OPEN_REDIRECT, etc.) core/features/ signatures - SCANNER_USER_AGENTS list core/detection/ @@ -35,9 +37,11 @@ from typing import NamedTuple from app.core.features.patterns import ( COMMAND_INJECTION, + CRLF_INJECTION, DOUBLE_ENCODED, FILE_INCLUSION, LOG4SHELL, + OPEN_REDIRECT, PATH_TRAVERSAL, SQLI, SSRF, @@ -77,6 +81,8 @@ _PATTERN_RULES: list[_PatternRule] = [ _PatternRule("FILE_INCLUSION", FILE_INCLUSION, 0.75), _PatternRule("SSRF", SSRF, 0.70), _PatternRule("PATH_TRAVERSAL", PATH_TRAVERSAL, 0.60), + _PatternRule("CRLF_INJECTION", CRLF_INJECTION, 0.65), + _PatternRule("OPEN_REDIRECT", OPEN_REDIRECT, 0.55), ] _THRESHOLD_RULES: list[_ThresholdRule] = [ diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/patterns.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/patterns.py index b499fcf8..dea8763c 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/patterns.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/patterns.py @@ -3,7 +3,7 @@ patterns.py Compiled regex patterns for web attack detection covering -7 OWASP categories plus encoding anomalies +9 OWASP categories plus encoding anomalies Defines case-insensitive compiled patterns for: SQLI (union select, sleep, benchmark, information_schema, hex @@ -17,9 +17,10 @@ backtick substitution, ${} expansion), FILE_INCLUSION (php://, file://, data://, phar:// wrapper schemes), SSRF (cloud metadata IPs 169.254.169.254, localhost with paths, dict:// and gopher://), LOG4SHELL (${jndi, ${lower, ${:- -patterns). Also provides ENCODED_CHARS, DOUBLE_ENCODED for -evasion detection, and ATTACK_COMBINED unioning all 7 -patterns +patterns), CRLF_INJECTION (%0d%0a sequences, bare CR/LF), +OPEN_REDIRECT (redirect params pointing to external URLs). +Also provides ENCODED_CHARS, DOUBLE_ENCODED for evasion +detection, and ATTACK_COMBINED unioning all 9 patterns Connects to: core/features/ @@ -115,6 +116,14 @@ _SSRF = ( _LOG4SHELL = r"(?:\$\{(?:j(?:ndi|ava)|lower|upper|:-|:\+|#))" +_CRLF_INJECTION = r"(?:%0[dD]%0[aA]|%0[dD]|%0[aA]|\\r\\n)" + +_OPEN_REDIRECT = ( + r"(?:(?:redirect|return|next|url|rurl|dest|destination|redir|" + r"redirect_uri|return_to|checkout_url|continue|return_path)" + r"\s*=\s*(?:https?://|//|\\\\)[^\s&]*)" +) + SQLI = re.compile(_SQLI, re.IGNORECASE) XSS = re.compile(_XSS, re.IGNORECASE) PATH_TRAVERSAL = re.compile(_PATH_TRAVERSAL, re.IGNORECASE) @@ -122,11 +131,14 @@ COMMAND_INJECTION = re.compile(_COMMAND_INJECTION, re.IGNORECASE) FILE_INCLUSION = re.compile(_FILE_INCLUSION, re.IGNORECASE) SSRF = re.compile(_SSRF, re.IGNORECASE) LOG4SHELL = re.compile(_LOG4SHELL, re.IGNORECASE) +CRLF_INJECTION = re.compile(_CRLF_INJECTION, re.IGNORECASE) +OPEN_REDIRECT = re.compile(_OPEN_REDIRECT, re.IGNORECASE) ATTACK_COMBINED = re.compile( r"|".join(( _SQLI, _XSS, _PATH_TRAVERSAL, _COMMAND_INJECTION, _FILE_INCLUSION, _SSRF, _LOG4SHELL, + _CRLF_INJECTION, _OPEN_REDIRECT, )), re.IGNORECASE, ) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py index 56862e47..82c89e5d 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py @@ -13,11 +13,13 @@ WindowAggregator, and encodes the merged 35-dim float vector. Stage 3 (_detection_worker): scores via RuleEngine, optionally runs ML ensemble inference (normalize AE/IF scores, fuse with configurable weights, blend with rule -score at 0.7 ML weight). Stage 4 (_dispatch_worker): -forwards ScoredRequests via the on_result callback. Stages -are connected by sized asyncio.Queues with poison-pill -shutdown propagation. EnrichedRequest and ScoredRequest -dataclasses carry data between stages +score at 0.7 ML weight), and persists per-model ML scores +on ScoredRequest for downstream observability. Stage 4 +(_dispatch_worker): forwards ScoredRequests via the +on_result callback. Stages are connected by sized +asyncio.Queues with poison-pill shutdown propagation. +EnrichedRequest and ScoredRequest dataclasses carry data +between stages Connects to: core/ingestion/parsers - parse_combined @@ -95,6 +97,7 @@ class ScoredRequest: rule_result: RuleResult final_score: float = 0.0 detection_mode: str = "rules" + ml_scores: dict[str, float] | None = None class Pipeline: @@ -138,6 +141,23 @@ class Pipeline: self._inference_engine = inference_engine self._ensemble_weights = ensemble_weights or DEFAULT_ENSEMBLE_WEIGHTS self._tasks: list[asyncio.Task[None]] = [] + self._stats: dict[str, int] = { + "parsed": 0, + "parse_errors": 0, + "enriched": 0, + "enrich_errors": 0, + "scored": 0, + "score_errors": 0, + "dispatched": 0, + "dispatch_errors": 0, + } + + @property + def stats(self) -> dict[str, int]: + """ + Return a snapshot of per-stage processed/error counters + """ + return dict(self._stats) async def start(self) -> None: """ @@ -174,7 +194,9 @@ class Pipeline: entry = parse_combined(line) if entry is not None: await self._parsed_queue.put(entry) + self._stats["parsed"] += 1 except Exception: + self._stats["parse_errors"] += 1 logger.exception("Parse error") self.raw_queue.task_done() @@ -222,7 +244,9 @@ class Pipeline: feature_vector=vector, geo=geo, )) + self._stats["enriched"] += 1 except Exception: + self._stats["enrich_errors"] += 1 logger.exception( "Feature extraction failed for %s", entry.ip, @@ -248,14 +272,17 @@ class Pipeline: final_score = rule_result.threat_score detection_mode = "rules" + per_model_scores: dict[str, float] | None = None if (self._inference_engine is not None and self._inference_engine.is_loaded and np is not None): - ml_scores = self._score_with_ml(enriched.feature_vector) - if ml_scores is not None: + per_model_scores = self._score_with_ml( + enriched.feature_vector, + ) + if per_model_scores is not None: ml_fused = fuse_scores( - ml_scores, + per_model_scores, self._ensemble_weights, ) final_score = blend_scores( @@ -273,8 +300,11 @@ class Pipeline: rule_result=rule_result, final_score=final_score, detection_mode=detection_mode, + ml_scores=per_model_scores, )) + self._stats["scored"] += 1 except Exception: + self._stats["score_errors"] += 1 logger.exception("Detection failed") self._feature_queue.task_done() @@ -314,7 +344,9 @@ class Pipeline: try: if self._on_result is not None: await self._on_result(scored) + self._stats["dispatched"] += 1 except Exception: + self._stats["dispatch_errors"] += 1 logger.exception( "Dispatch failed for %s", scored.entry.ip, diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py index 38e56b25..eb18ddc4 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py @@ -3,18 +3,22 @@ tailer.py Watchdog-based nginx log file tailer with rotation -detection pushing raw lines into an asyncio queue +detection and position persistence pushing raw lines +into an asyncio queue _LogHandler extends FileSystemEventHandler to tail a single -target file: _open_target seeks to EOF, on_modified reads -new lines via _read_new_lines and checks inode changes for -rotation, on_moved handles rename-based rotation (access -.log -> access.log.1), on_created handles new-file -rotation. Lines are pushed via call_soon_threadsafe into -the asyncio queue, with QueueFull drops logged. LogTailer -wraps _LogHandler with a PollingObserver (2s interval) -watching the target's parent directory, providing start/ -stop lifecycle and is_active property +target file: _open_target restores position from a JSON +position file (inode + offset) if available, otherwise +seeks to EOF. on_modified reads new lines via +_read_new_lines and checks inode changes for rotation, +on_moved handles rename-based rotation (access.log -> +access.log.1), on_created handles new-file rotation. +_save_position persists the current inode and file offset +after each batch of reads. Lines are pushed via +call_soon_threadsafe into the asyncio queue, with QueueFull +drops logged. LogTailer wraps _LogHandler with a +PollingObserver (2s interval) watching the target's parent +directory, providing start/stop lifecycle and is_active Connects to: factory.py - started/stopped in lifespan @@ -24,6 +28,7 @@ Connects to: """ import asyncio +import json import logging import os from io import TextIOWrapper @@ -51,6 +56,7 @@ class _LogHandler(FileSystemEventHandler): target: str, queue: asyncio.Queue[str | None], loop: asyncio.AbstractEventLoop, + position_path: Path | None = None, ) -> None: super().__init__() self._target = target @@ -58,24 +64,84 @@ class _LogHandler(FileSystemEventHandler): self._loop = loop self._file: TextIOWrapper | None = None self._inode: int | None = None + self._position_path = position_path self._open_target() def _open_target(self) -> None: """ - Open the target log file and seek to the end. + Open the target log file, restoring saved position + if the inode matches, otherwise seeking to EOF """ try: self._file = open( # noqa: SIM115 self._target, encoding="utf-8", errors="replace") - self._file.seek(0, os.SEEK_END) self._inode = os.stat(self._target).st_ino - logger.info("Tailing %s (inode %s)", self._target, self._inode) + + saved = self._load_position() + if ( + saved is not None + and saved["inode"] == self._inode + ): + self._file.seek(saved["offset"]) + logger.info( + "Tailing %s (inode %s) resumed at offset %d", + self._target, + self._inode, + saved["offset"], + ) + else: + self._file.seek(0, os.SEEK_END) + logger.info( + "Tailing %s (inode %s) from EOF", + self._target, + self._inode, + ) except FileNotFoundError: - logger.warning("Log file %s not found — waiting for creation", - self._target) + logger.warning( + "Log file %s not found — waiting for creation", + self._target, + ) self._file = None self._inode = None + def _load_position(self) -> dict[str, int] | None: + """ + Read saved (inode, offset) from the position file + """ + if self._position_path is None: + return None + try: + data = json.loads( + self._position_path.read_text(encoding="utf-8"), + ) + return { + "inode": data["inode"], + "offset": data["offset"], + } + except (FileNotFoundError, KeyError, json.JSONDecodeError): + return None + + def _save_position(self) -> None: + """ + Persist current inode and file offset to disk + """ + if ( + self._position_path is None + or self._file is None + or self._inode is None + ): + return + try: + self._position_path.write_text( + json.dumps({ + "inode": self._inode, + "offset": self._file.tell(), + }), + encoding="utf-8", + ) + except OSError: + logger.debug("Failed to save tailer position") + def _enqueue(self, line: str) -> None: """ Push one line into the queue, logging drops on full queue @@ -87,7 +153,7 @@ class _LogHandler(FileSystemEventHandler): def _read_new_lines(self) -> None: """ - Read all new complete lines from the current file position. + Read all new complete lines from the current file position """ if self._file is None: return @@ -97,6 +163,8 @@ class _LogHandler(FileSystemEventHandler): if stripped: self._loop.call_soon_threadsafe(self._enqueue, stripped) + self._save_position() + def _handle_rotation(self) -> None: """ Finish reading the old file, then reopen the target at position 0. @@ -188,9 +256,12 @@ class LogTailer: log_path: str, queue: asyncio.Queue[str | None], loop: asyncio.AbstractEventLoop, + position_path: Path | None = None, ) -> None: self._log_path = log_path - self._handler = _LogHandler(log_path, queue, loop) + self._handler = _LogHandler( + log_path, queue, loop, position_path, + ) self._observer = PollingObserver(timeout=2) self._started = False diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py b/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py index 5b904601..1d2f76b6 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py @@ -119,7 +119,13 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: log_dir = Path(settings.nginx_log_path).resolve().parent if log_dir.is_dir(): loop = asyncio.get_running_loop() - tailer = LogTailer(settings.nginx_log_path, pipeline.raw_queue, loop) + position_path = Path(settings.model_dir) / ".tailer_pos.json" + tailer = LogTailer( + settings.nginx_log_path, + pipeline.raw_queue, + loop, + position_path=position_path, + ) tailer.start() else: logger.warning("Log directory %s not found — tailer disabled", log_dir) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py b/PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py index 30396781..1ba45a73 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py @@ -151,7 +151,10 @@ async def create_threat_event( user_agent=scored.entry.user_agent, threat_score=scored.final_score, severity=classify_severity(scored.final_score), - component_scores=scored.rule_result.component_scores, + component_scores={ + **scored.rule_result.component_scores, + **(scored.ml_scores or {}), + }, geo_country=(scored.geo.country if scored.geo else None), geo_city=(scored.geo.city if scored.geo else None), geo_lat=(scored.geo.lat if scored.geo else None), diff --git a/PROJECTS/advanced/ai-threat-detection/backend/data/models/ae.onnx b/PROJECTS/advanced/ai-threat-detection/backend/data/models/ae.onnx new file mode 100644 index 00000000..f07aa47a Binary files /dev/null and b/PROJECTS/advanced/ai-threat-detection/backend/data/models/ae.onnx differ diff --git a/PROJECTS/advanced/ai-threat-detection/backend/data/models/ae.onnx.data b/PROJECTS/advanced/ai-threat-detection/backend/data/models/ae.onnx.data new file mode 100644 index 00000000..0c1afc35 Binary files /dev/null and b/PROJECTS/advanced/ai-threat-detection/backend/data/models/ae.onnx.data differ diff --git a/PROJECTS/advanced/ai-threat-detection/backend/data/models/if.onnx b/PROJECTS/advanced/ai-threat-detection/backend/data/models/if.onnx new file mode 100644 index 00000000..e0ca087f Binary files /dev/null and b/PROJECTS/advanced/ai-threat-detection/backend/data/models/if.onnx differ diff --git a/PROJECTS/advanced/ai-threat-detection/backend/data/models/rf.onnx b/PROJECTS/advanced/ai-threat-detection/backend/data/models/rf.onnx new file mode 100644 index 00000000..558133f9 Binary files /dev/null and b/PROJECTS/advanced/ai-threat-detection/backend/data/models/rf.onnx differ diff --git a/PROJECTS/advanced/ai-threat-detection/backend/data/models/scaler.json b/PROJECTS/advanced/ai-threat-detection/backend/data/models/scaler.json new file mode 100644 index 00000000..69255df9 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/data/models/scaler.json @@ -0,0 +1,114 @@ +{ + "center": [ + 1.0, + 1.0, + 3.057476043701172, + 12.0, + 0.0, + 0.0, + 0.0, + 0.0, + 200.0, + 2.0, + 24034.5, + 11.5, + 6.0, + 1.0, + 114.0, + 5.101086616516113, + 0.0, + 0.0, + 0.0, + 0.1818181872367859, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "scale": [ + 1.0, + 2.0, + 0.8716020584106445, + 9.0, + 6.0, + 1.0, + 1.0, + 1.0, + 101.0, + 1.0, + 22906.75, + 12.75, + 1.0, + 1.0, + 45.0, + 0.3695187568664551, + 1.0, + 1.0, + 1.0, + 0.06043955683708191, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "n_features": 35, + "feature_names": [ + "http_method", + "path_depth", + "path_entropy", + "path_length", + "query_string_length", + "query_param_count", + "has_encoded_chars", + "has_double_encoding", + "status_code", + "status_class", + "response_size", + "hour_of_day", + "day_of_week", + "is_weekend", + "ua_length", + "ua_entropy", + "is_known_bot", + "is_known_scanner", + "has_attack_pattern", + "special_char_ratio", + "file_extension", + "country_code", + "is_private_ip", + "req_count_1m", + "req_count_5m", + "req_count_10m", + "error_rate_5m", + "unique_paths_5m", + "unique_uas_10m", + "method_entropy_5m", + "avg_response_size_5m", + "status_diversity_5m", + "path_depth_variance_5m", + "inter_request_time_mean", + "inter_request_time_std" + ] +} diff --git a/PROJECTS/advanced/ai-threat-detection/backend/data/models/threshold.json b/PROJECTS/advanced/ai-threat-detection/backend/data/models/threshold.json new file mode 100644 index 00000000..08c5e102 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/data/models/threshold.json @@ -0,0 +1,3 @@ +{ + "threshold": 0.9133416414260864 +} diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/orchestrator.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/orchestrator.py index 6a562d6a..53566a67 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/ml/orchestrator.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/orchestrator.py @@ -51,6 +51,8 @@ from ml.train_classifiers import ( ) from ml.validation import ValidationResult, validate_ensemble +from app.core.features.mappings import FEATURE_ORDER + logger = logging.getLogger(__name__) N_FEATURES = 35 @@ -240,7 +242,10 @@ class TrainingOrchestrator: ae_result["model"], self._output_dir / AE_FILENAME, ) - ae_result["scaler"].save_json(self._output_dir / SCALER_FILENAME) + ae_result["scaler"].save_json( + self._output_dir / SCALER_FILENAME, + feature_names=list(FEATURE_ORDER), + ) threshold_data = {"threshold": ae_result["threshold"]} (self._output_dir / THRESHOLD_FILENAME).write_text( json.dumps(threshold_data, indent=2)) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/scaler.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/scaler.py index 695d2812..25795c93 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/ml/scaler.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/scaler.py @@ -9,9 +9,11 @@ FeatureScaler wraps sklearn RobustScaler (median/IQR normalization) to handle outlier-heavy HTTP traffic data. Provides fit, transform, fit_transform, and inverse_transform mirroring the sklearn API. save_json -serializes center and scale arrays to a human-readable -JSON file (avoiding pickle for security and portability), -and load_json reconstructs a fitted scaler from that file. +serializes center, scale arrays, and optional feature_names +to a human-readable JSON file (avoiding pickle for security +and portability), and load_json reconstructs a fitted +scaler from that file with optional feature ordering +validation against expected_feature_names. Only the autoencoder uses this scaler since tree-based models (random forest, isolation forest) are scale-invariant @@ -84,25 +86,47 @@ class FeatureScaler: self.fit(X) return self.transform(X) - def save_json(self, path: Path | str) -> None: + def save_json( + self, + path: Path | str, + feature_names: list[str] | None = None, + ) -> None: """ Serialize scaler parameters to a human-readable JSON file. """ if not self._fitted or self._scaler is None: raise RuntimeError("Scaler has not been fitted") - data = { + data: dict[str, object] = { "center": self._scaler.center_.tolist(), "scale": self._scaler.scale_.tolist(), "n_features": int(self._scaler.n_features_in_), } + if feature_names is not None: + data["feature_names"] = feature_names Path(path).write_text(json.dumps(data, indent=2), encoding="utf-8") @classmethod - def load_json(cls, path: Path | str) -> FeatureScaler: + def load_json( + cls, + path: Path | str, + expected_feature_names: list[str] | None = None, + ) -> FeatureScaler: """ Reconstruct a fitted scaler from a JSON file. """ data = json.loads(Path(path).read_text(encoding="utf-8")) + + stored_names = data.get("feature_names") + if ( + expected_feature_names is not None + and stored_names is not None + and stored_names != expected_feature_names + ): + raise ValueError( + "Feature ordering mismatch between trained " + "scaler and current FEATURE_ORDER" + ) + scaler = cls() scaler._scaler = RobustScaler() scaler._scaler.center_ = np.array(data["center"], dtype=np.float64) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/conftest.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/conftest.py index 9a306b2b..0dad33ed 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/conftest.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/conftest.py @@ -99,6 +99,7 @@ async def db_client(db_engine) -> AsyncIterator[AsyncClient]: await session.commit() test_app.dependency_overrides[get_session] = override_get_session + test_app.state.session_factory = factory transport = ASGITransport(app=test_app) async with AsyncClient(transport=transport, diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_detection.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_detection.py index 22b0bebc..a5c84a7b 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_detection.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_detection.py @@ -15,8 +15,9 @@ rules aggregate to higher scores, scores are clamped to [0, 1], severity thresholds align with architecture (LOW < 0.5, MEDIUM >= 0.5, HIGH >= 0.7), component_scores match matched_rules, FILE_INCLUSION detects PHP stream -wrappers, and DOUBLE_ENCODING detects %25-prefixed -sequences +wrappers, DOUBLE_ENCODING detects %25-prefixed sequences, +CRLF_INJECTION detects %0d%0a header injection, and +OPEN_REDIRECT detects redirect params to external URLs Connects to: core/detection/rules - RuleEngine @@ -252,3 +253,29 @@ def test_double_encoding() -> None: entry = _make_entry(path="/path%2527trick") result = ENGINE.score_request(_empty_windowed(), entry) assert "DOUBLE_ENCODING" in result.matched_rules + + +def test_crlf_injection() -> None: + """ + CRLF sequences in the URI trigger the CRLF_INJECTION rule + """ + entry = _make_entry( + path="/api/v1/users", + query_string="name=admin%0d%0aSet-Cookie:session=hijacked", + ) + result = ENGINE.score_request(_empty_windowed(), entry) + assert "CRLF_INJECTION" in result.matched_rules + assert result.threat_score >= 0.5 + + +def test_open_redirect() -> None: + """ + Redirect parameter pointing to an external URL triggers OPEN_REDIRECT + """ + entry = _make_entry( + path="/login", + query_string="redirect_uri=https://evil.com/steal", + ) + result = ENGINE.score_request(_empty_windowed(), entry) + assert "OPEN_REDIRECT" in result.matched_rules + assert result.threat_score >= 0.5 diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/App.tsx b/PROJECTS/advanced/ai-threat-detection/frontend/src/App.tsx index cfdc5a6b..a14d7147 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/App.tsx +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/App.tsx @@ -7,8 +7,8 @@ // Wraps the application in QueryClientProvider (TanStack // React Query), provides the browser router via // RouterProvider, renders a dark-themed Sonner toast -// container at top-right, and includes ReactQueryDevtools -// in development mode +// container at top-right styled by toast.scss design tokens, +// and includes ReactQueryDevtools in development mode // =================== import { QueryClientProvider } from '@tanstack/react-query' @@ -18,24 +18,14 @@ import { Toaster } from 'sonner' import { queryClient } from '@/core/api' import { router } from '@/core/app/routers' +import '@/core/app/toast.scss' export default function App(): React.ReactElement { return (
- +
diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/components/alert-feed.module.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/components/alert-feed.module.scss index 09f904ed..db5483ba 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/components/alert-feed.module.scss +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/components/alert-feed.module.scss @@ -1,9 +1,9 @@ // =================== // © AngelaMos | 2026 // alert-feed.module.scss -// + // CSS module for the live alert feed component -// + // Styles the feed container with surface background and // muted border, a header row with title and connection // status dot (green connected, red disconnected), and a diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/components/method-badge.module.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/components/method-badge.module.scss index 04f45201..6d1da921 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/components/method-badge.module.scss +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/components/method-badge.module.scss @@ -1,9 +1,9 @@ // =================== // © AngelaMos | 2026 // method-badge.module.scss -// + // CSS module for the HTTP method badge component -// + // Styles a monospace semibold 2xs text badge with wide // letter spacing and a default $text-lighter color. Seven // method variant classes (.get through .options) apply diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/components/severity-badge.module.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/components/severity-badge.module.scss index d1cc952e..37d5bdeb 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/components/severity-badge.module.scss +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/components/severity-badge.module.scss @@ -1,9 +1,9 @@ // =================== // © AngelaMos | 2026 // severity-badge.module.scss -// + // CSS module for the severity badge component -// + // Styles a compact inline-flex pill badge with full border // radius, 2xs semibold uppercase text, and wider letter // spacing. Three severity variants map to token pairs: diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/components/stat-card.module.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/components/stat-card.module.scss index e7557319..64ab6b0f 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/components/stat-card.module.scss +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/components/stat-card.module.scss @@ -1,9 +1,9 @@ // =================== // © AngelaMos | 2026 // stat-card.module.scss -// + // CSS module for the dashboard metric card component -// + // Styles a column flex card with surface background, muted // border, and large border radius. Displays a 3xl semibold // value with tight letter spacing, an sm lighter label, and diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/components/threat-detail.module.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/components/threat-detail.module.scss index c3f991ae..f62d39cd 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/components/threat-detail.module.scss +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/components/threat-detail.module.scss @@ -1,9 +1,9 @@ // =================== // © AngelaMos | 2026 // threat-detail.module.scss -// + // CSS module for the threat detail slide-in panel -// + // Styles a full-height modal overlay (60% opacity black, // z-modal) with a right-aligned 520px max-width panel. // Contains a sticky header with close button, sectioned diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/components/threat-detail.tsx b/PROJECTS/advanced/ai-threat-detection/frontend/src/components/threat-detail.tsx index ef79d97b..0a65c5ca 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/components/threat-detail.tsx +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/components/threat-detail.tsx @@ -4,7 +4,8 @@ // // Modal dialog for full threat event inspection // -// Renders a click-to-dismiss overlay with a detail panel +// Renders a click-to-dismiss overlay (also dismissable via +// Escape key) with a detail panel // displaying four sections: Overview (severity badge, // threat score to 4 decimals, detection timestamp, review // status), Request (source IP, method, path, status code, @@ -16,6 +17,7 @@ // pages/threats // =================== +import { useEffect } from 'react' import { LuX } from 'react-icons/lu' import type { ThreatEvent } from '@/api/types' import { SeverityBadge } from './severity-badge' @@ -34,6 +36,15 @@ export function ThreatDetail({ threat, onClose, }: ThreatDetailProps): React.ReactElement | null { + useEffect(() => { + if (!threat) return + const handler = (e: KeyboardEvent) => { + if (e.key === 'Escape') onClose() + } + document.addEventListener('keydown', handler) + return () => document.removeEventListener('keydown', handler) + }, [threat, onClose]) + if (!threat) return null return ( diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/core/app/shell.module.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/core/app/shell.module.scss index 1fc56c64..96eb6b2b 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/core/app/shell.module.scss +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/core/app/shell.module.scss @@ -1,10 +1,10 @@ // =================== // © AngelaMos | 2026 // shell.module.scss -// + // Application shell layout styles with collapsible sidebar // and responsive mobile drawer -// + // Styles a fixed-position sidebar (240px expanded, 64px // collapsed) with dot-grid background, NavLink items with // hover and active states, red-tinted nav icons, a chevron diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/core/app/toast.module.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/core/app/toast.module.scss deleted file mode 100644 index a61950a9..00000000 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/core/app/toast.module.scss +++ /dev/null @@ -1,79 +0,0 @@ -// =================== -// © AngelaMos | 2026 -// toast.module.scss -// -// Sonner toast notification theme overrides -// -// Applies global dark theme CSS custom properties to -// [data-sonner-toaster] for normal, success, error, -// warning, and info variants using $bg-surface-100, -// $border-default, and $text-default tokens. Styles -// [data-sonner-toast] with $radius-md border radius, -// medium-weight titles, light-colored descriptions, and -// hover-brightened close buttons. Error toasts receive a -// $error-default border accent. Connects to App.tsx, -// styles/_index -// =================== - -@use '@/styles' as *; - -:global { - [data-sonner-toaster] { - --normal-bg: #{$bg-surface-100}; - --normal-border: #{$border-default}; - --normal-text: #{$text-default}; - - --success-bg: #{$bg-surface-100}; - --success-border: #{$border-default}; - --success-text: #{$text-default}; - - --error-bg: #{$bg-surface-100}; - --error-border: #{$error-default}; - --error-text: #{$text-default}; - - --warning-bg: #{$bg-surface-100}; - --warning-border: #{$border-default}; - --warning-text: #{$text-default}; - - --info-bg: #{$bg-surface-100}; - --info-border: #{$border-default}; - --info-text: #{$text-default}; - - font-family: $font-sans; - } - - [data-sonner-toast] { - border-radius: $radius-md; - padding: $space-3 $space-4; - font-size: $font-size-sm; - border: 1px solid $border-default; - background: $bg-surface-100; - color: $text-default; - - [data-title] { - font-weight: $font-weight-medium; - } - - [data-description] { - color: $text-light; - font-size: $font-size-xs; - } - - [data-close-button] { - background: none; - border: none; - padding: 0; - cursor: pointer; - color: $text-muted; - @include transition-fast; - - @include hover { - color: $text-default; - } - } - } - - [data-sonner-toast][data-type='error'] { - border-color: $error-default; - } -} diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/core/app/toast.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/core/app/toast.scss new file mode 100644 index 00000000..df9fca3b --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/core/app/toast.scss @@ -0,0 +1,76 @@ +// =================== +// © AngelaMos | 2026 +// toast.scss + +// Sonner toast notification theme overrides + +// Applies global dark theme CSS custom properties to +// [data-sonner-toaster] for normal, success, error, +// warning, and info variants using $bg-surface-100, +// $border-default, and $text-default tokens. Styles +// [data-sonner-toast] with $radius-md border radius, +// medium-weight titles, light-colored descriptions, and +// hover-brightened close buttons. Error toasts receive a +// $error-default border accent. Connects to styles.scss +// =================== + +@use '@/styles' as *; + +[data-sonner-toaster] { + --normal-bg: #{$bg-surface-100}; + --normal-border: #{$border-default}; + --normal-text: #{$text-default}; + + --success-bg: #{$bg-surface-100}; + --success-border: #{$border-default}; + --success-text: #{$text-default}; + + --error-bg: #{$bg-surface-100}; + --error-border: #{$error-default}; + --error-text: #{$text-default}; + + --warning-bg: #{$bg-surface-100}; + --warning-border: #{$border-default}; + --warning-text: #{$text-default}; + + --info-bg: #{$bg-surface-100}; + --info-border: #{$border-default}; + --info-text: #{$text-default}; + + font-family: $font-sans; +} + +[data-sonner-toast] { + border-radius: $radius-md; + padding: $space-3 $space-4; + font-size: $font-size-sm; + border: 1px solid $border-default; + background: $bg-surface-100; + color: $text-default; + + [data-title] { + font-weight: $font-weight-medium; + } + + [data-description] { + color: $text-light; + font-size: $font-size-xs; + } + + [data-close-button] { + background: none; + border: none; + padding: 0; + cursor: pointer; + color: $text-muted; + @include transition-fast; + + @include hover { + color: $text-default; + } + } +} + +[data-sonner-toast][data-type='error'] { + border-color: $error-default; +} diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/dashboard/dashboard.module.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/dashboard/dashboard.module.scss index 89e295f4..f0f46bf1 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/dashboard/dashboard.module.scss +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/dashboard/dashboard.module.scss @@ -1,9 +1,9 @@ // =================== // © AngelaMos | 2026 // dashboard.module.scss -// + // CSS module for the dashboard page layout -// + // Styles a padded column flex page with a 4-column // responsive stat row (collapsing to 2 then 1 at lg/sm // breakpoints), a severity section with a proportional diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/models/models.module.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/models/models.module.scss index 3f6d899b..3fb27b99 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/models/models.module.scss +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/models/models.module.scss @@ -1,9 +1,9 @@ // =================== // © AngelaMos | 2026 // models.module.scss -// + // CSS module for the models management page -// + // Styles a status banner with loaded (green) and not-loaded // (amber) severity-themed variants, a retrain button with // accent-muted background and spinning icon animation during diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/threats/index.tsx b/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/threats/index.tsx index cc5ad6fa..2e239697 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/threats/index.tsx +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/threats/index.tsx @@ -7,8 +7,9 @@ // // Exports a lazy-loaded Component (displayName ThreatsPage) // that manages offset, severity filter (ALL/HIGH/MEDIUM/ -// LOW), and source IP text filter as local state. Passes -// these as ThreatParams to useThreats with PAGINATION +// LOW), and source IP text filter as local state. The IP +// filter uses useDeferredValue to avoid per-keystroke API +// calls. Passes these as ThreatParams to useThreats with PAGINATION // defaults. Renders a responsive table with time, source // IP, MethodBadge, path, score to 3 decimals, // SeverityBadge, and status code columns. Rows are @@ -19,7 +20,7 @@ // components/threat-detail, config // =================== -import { useState } from 'react' +import { useDeferredValue, useState } from 'react' import { useThreats } from '@/api/hooks' import type { ThreatEvent } from '@/api/types' import { MethodBadge, SeverityBadge, ThreatDetail } from '@/components' @@ -37,12 +38,13 @@ export function Component(): React.ReactElement { const [severity, setSeverity] = useState('ALL') const [sourceIp, setSourceIp] = useState('') const [selectedThreat, setSelectedThreat] = useState(null) + const deferredIp = useDeferredValue(sourceIp) const params = { limit: PAGINATION.DEFAULT_LIMIT, offset, ...(severity !== 'ALL' && { severity }), - ...(sourceIp && { source_ip: sourceIp }), + ...(deferredIp && { source_ip: deferredIp }), } const { data, isLoading } = useThreats(params) diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/threats/threats.module.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/threats/threats.module.scss index 3c987b86..8497b468 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/threats/threats.module.scss +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/pages/threats/threats.module.scss @@ -1,9 +1,9 @@ // =================== // © AngelaMos | 2026 // threats.module.scss -// + // CSS module for the threats table page -// + // Styles a filter bar with select dropdown and text input // (accent-bordered on focus), a scrollable table wrapper // with uppercase column headers, clickable rows with hover diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/styles.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/styles.scss index 69d6b8e5..565604aa 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/styles.scss +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/styles.scss @@ -1,9 +1,9 @@ // =================== // © AngelaMos | 2026 // styles.scss -// + // Global stylesheet entry point with root element setup -// + // Forwards tokens, fonts, and mixins for downstream module // consumption, applies the CSS reset, and defines #root as // a full-viewport column flex container with $bg-default diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_index.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_index.scss index 506508ee..b14937da 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_index.scss +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_index.scss @@ -1,9 +1,9 @@ // =================== // © AngelaMos | 2026 // _index.scss -// + // SCSS module forwarding index for the styles package -// + // Forwards tokens, fonts, and mixins so that downstream // SCSS modules can access the full design system with a // single @use '@/styles' as * import diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_mixins.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_mixins.scss index ed25c9b5..b273c672 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_mixins.scss +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_mixins.scss @@ -1,10 +1,10 @@ // =================== // © AngelaMos | 2026 // _mixins.scss -// + // Reusable SCSS mixins for layout, typography, transitions, // and responsiveness -// + // Defines a $breakpoints map (xs through 2xl) with // breakpoint-up and breakpoint-down media query mixins, // flex layout helpers (flex-center, flex-between, diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_reset.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_reset.scss index cb8a1e0a..beeccd92 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_reset.scss +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_reset.scss @@ -35,8 +35,6 @@ html { font-size: 16px; - -moz-text-size-adjust: none; - -webkit-text-size-adjust: none; text-size-adjust: none; overflow-x: hidden; } diff --git a/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_tokens.scss b/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_tokens.scss index efac199b..64228e15 100644 --- a/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_tokens.scss +++ b/PROJECTS/advanced/ai-threat-detection/frontend/src/styles/_tokens.scss @@ -1,9 +1,9 @@ // =================== // © AngelaMos | 2026 // _tokens.scss -// + // Design token variables for the entire UI system -// + // Defines the full token set consumed by all SCSS modules: // 8px-base spacing scale ($space-0 through $space-32), // typography scale (3xs through 5xl), font weights (regular, diff --git a/PROJECTS/advanced/ai-threat-detection/learn/00-OVERVIEW.md b/PROJECTS/advanced/ai-threat-detection/learn/00-OVERVIEW.md new file mode 100644 index 00000000..283caef7 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/learn/00-OVERVIEW.md @@ -0,0 +1,160 @@ +# AngelusVigil: AI Threat Detection + +## What This Is + +AngelusVigil is a real-time threat detection engine that analyzes nginx web server logs using a 3-model ML ensemble (autoencoder + random forest + isolation forest) to classify HTTP traffic as benign or malicious. It deploys as a dockerized sidecar alongside any nginx-based infrastructure with zero code changes required. The system processes logs through a 4-stage async pipeline, extracts 35-dimensional feature vectors, scores traffic using both rule-based and ML detection, and surfaces alerts through a React dashboard with live WebSocket updates. + +## Why This Matters + +Web application firewalls (WAFs) catch known attack patterns, but they miss novel attacks and struggle with behavioral anomalies. In the 2017 Equifax breach (CVE-2017-5638), attackers exploited an Apache Struts vulnerability that signature-based tools missed for months. The 2021 Log4Shell vulnerability (CVE-2021-44228) spread across millions of servers before WAF rules were updated. The Capital One breach in 2019 involved a misconfigured WAF that an insider exploited to exfiltrate 100 million customer records. + +ML-based detection fills these gaps by learning what "normal" looks like for your specific traffic and flagging deviations. This project builds that system from scratch. + +**Real world scenarios where this applies:** +- A SaaS platform running nginx as a reverse proxy wants to detect SQL injection, path traversal, and credential stuffing without buying a commercial WAF +- A DevSecOps team needs real time alerting when attack patterns spike against their API endpoints, with enough context (GeoIP, feature vectors, matched rules) to triage quickly +- A security team wants to combine signature based detection (day 0 coverage) with ML models that improve over time as they label true/false positives from production traffic + +## What You'll Learn + +This project teaches you how ML-powered threat detection works at the infrastructure level. By building it yourself, you'll understand: + +**Security Concepts:** +- Anomaly detection using autoencoders trained exclusively on normal traffic, so anything the model can't reconstruct well is suspicious +- Ensemble learning where multiple models vote on whether a request is malicious, reducing false positives from any single model +- Cold-start detection using ModSecurity CRS-inspired rules that provide immediate coverage before ML models are trained +- Feature engineering for HTTP traffic, turning raw log lines into 35-dimensional numeric vectors that capture request structure, behavioral patterns, and temporal signals + +**Technical Skills:** +- Building async data pipelines with backpressure using `asyncio.Queue` and poison-pill shutdown propagation +- Training and exporting PyTorch and scikit-learn models to ONNX format for fast CPU inference +- Implementing sliding window aggregation with Redis sorted sets for per-IP behavioral features +- Writing a Watchdog-based file tailer that handles nginx log rotation without missing lines + +**Tools and Techniques:** +- ONNX Runtime for cross-framework model serving (27% faster than native PyTorch inference) +- MLflow experiment tracking for model versioning and metric logging +- Redis pub/sub for real-time WebSocket relay across multiple backend workers +- Docker Compose orchestration of a 5-service production stack + +## Prerequisites + +Before starting, you should understand: + +**Required knowledge:** +- Python async/await (you'll work with `asyncio.Queue`, `asyncio.Task`, and async context managers throughout the pipeline) +- Basic machine learning concepts (what training/inference means, what loss functions do, the difference between supervised and unsupervised learning) +- HTTP fundamentals (request methods, status codes, query strings, headers, how nginx access logs are structured) +- Docker and Docker Compose (the entire system runs as containers) + +**Tools you'll need:** +- Docker and Docker Compose v2 +- Python 3.13+ (the backend uses modern type syntax like `dict[str, float]` and `X | None`) +- Node.js 20+ and pnpm (for the React frontend) +- just (command runner, like make but better) +- uv (Python package manager) + +**Helpful but not required:** +- Experience with FastAPI or any async Python web framework +- Familiarity with PyTorch or scikit-learn +- Understanding of Redis data structures (sorted sets, pub/sub) + +## Quick Start + +Get the project running locally: + +```bash +cd PROJECTS/advanced/ai-threat-detection + +# Install just if you don't have it +# https://github.com/casey/just + +# One-time setup: install deps and create .env +just setup + +# Edit .env with your values (at minimum set POSTGRES_PASSWORD and API_KEY) +# GEOIP credentials are optional for local dev + +# Start the dev stack (postgres, redis, backend with hot-reload, frontend) +just dev-up + +# In another terminal, start the dev log generator to simulate traffic +just devlog-up + +# Generate some mixed traffic (normal + attacks) +just devlog-simulate mixed 200 +``` + +Expected output: Open `http://localhost:5173` in your browser. You should see the dashboard with stat cards showing detected threats, a severity distribution bar, a live alert feed receiving WebSocket events, and ranked lists of top attacker IPs and most targeted paths. + +The backend starts in rules-only mode. To enable ML detection, train the models: + +```bash +just vigil-train +``` + +After training completes (about 2 minutes with synthetic data), restart the backend and it will load the ONNX models and switch to hybrid detection mode. + +## Project Structure + +``` +ai-threat-detection/ +├── backend/ +│ ├── app/ +│ │ ├── api/ # FastAPI route handlers (health, threats, stats, models, ws) +│ │ ├── core/ +│ │ │ ├── alerts/ # AlertDispatcher: store + publish scored events +│ │ │ ├── detection/ # RuleEngine, InferenceEngine, ensemble scoring +│ │ │ ├── enrichment/ # GeoIP lookups via MaxMind +│ │ │ ├── features/ # Feature extraction, Redis aggregation, encoding +│ │ │ └── ingestion/ # Log tailer, parsers, 4-stage pipeline +│ │ ├── models/ # SQLModel ORM (ThreatEvent, ModelMetadata) +│ │ ├── schemas/ # Pydantic request/response models +│ │ ├── services/ # Database query logic (threat_service, stats_service) +│ │ ├── config.py # Pydantic Settings (env vars, defaults, validation) +│ │ └── factory.py # App factory with async lifespan +│ ├── ml/ # Training pipeline (orchestrator, splitting, export, validation) +│ ├── cli/ # Typer CLI (train, retrain, replay commands) +│ ├── tests/ # pytest suite (parsers, features, detection, integration) +│ └── alembic/ # Database migrations +├── frontend/ +│ ├── src/ +│ │ ├── api/ # React Query hooks, Zod schemas, Axios client +│ │ ├── components/ # AlertFeed, SeverityBadge, MethodBadge, StatCard, ThreatDetail +│ │ ├── core/ # Router config, shell layout, query client +│ │ └── pages/ # Dashboard, Threats (table + filters), Models +│ └── vite.config.ts +├── dev-log/ # Synthetic nginx traffic generator for testing +├── infra/ # Dockerfiles, redis.conf, nginx.conf +├── compose.yml # Production 5-service stack +├── dev.compose.yml # Development stack with hot-reload +└── justfile # Task runner commands +``` + +## Next Steps + +1. **Understand the concepts** - Read [01-CONCEPTS.md](./01-CONCEPTS.md) to learn anomaly detection, ensemble methods, and feature engineering for HTTP traffic +2. **Study the architecture** - Read [02-ARCHITECTURE.md](./02-ARCHITECTURE.md) to see how the 4-stage pipeline, Redis windowing, and ONNX inference fit together +3. **Walk through the code** - Read [03-IMPLEMENTATION.md](./03-IMPLEMENTATION.md) for a detailed walkthrough of the ingestion pipeline, rule engine, ML training, and frontend +4. **Extend the project** - Read [04-CHALLENGES.md](./04-CHALLENGES.md) for ideas like adding new model types, integrating with SIEM tools, or building an active learning workflow + +## Common Issues + +**Backend fails to start with "database connection refused"** +``` +sqlalchemy.exc.OperationalError: connection to server at "localhost" ... refused +``` +Solution: Make sure PostgreSQL is running. If using Docker, check `just dev-ps` to verify the postgres container is healthy. The backend depends on `postgres:service_healthy`, so it will wait, but if postgres itself failed to start, check the logs with `just dev-logs postgres`. + +**"onnxruntime not installed" warning at startup** +This is expected if you haven't installed the ML dependencies. The system falls back to rules-only mode automatically. To enable ML detection, run `cd backend && uv sync --group ml` and then train the models with `just vigil-train`. + +**WebSocket alerts not appearing on the dashboard** +Check that Redis is running (`just dev-logs redis`). The WebSocket relay depends on Redis pub/sub. Also verify the backend is processing logs by hitting `http://localhost:8000/health` and checking the `stats` field in the response. + +## Related Projects + +If you found this interesting, check out: +- [SIEM Dashboard](../../intermediate/siem-dashboard/) - Build a Security Information and Event Management dashboard with Flask, MongoDB, and Redis Streams for log aggregation and correlation +- [Honeypot Network](../honeypot-network/) - Deploy deceptive services that generate the kind of attack traffic this project detects +- [API Security Scanner](../../intermediate/api-security-scanner/) - Active vulnerability scanning that complements this project's passive detection approach diff --git a/PROJECTS/advanced/ai-threat-detection/learn/01-CONCEPTS.md b/PROJECTS/advanced/ai-threat-detection/learn/01-CONCEPTS.md new file mode 100644 index 00000000..533a5a8e --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/learn/01-CONCEPTS.md @@ -0,0 +1,445 @@ +# Core Security Concepts + +This document covers the security and machine learning concepts behind AngelusVigil. These aren't textbook definitions. We'll dig into how each concept is actually implemented in the codebase and why the design choices matter. + +## Anomaly Detection + +### What It Is + +Anomaly detection identifies data points that don't fit the expected pattern. In the context of web traffic, "expected" means the normal behavior of legitimate users and applications. Anything that deviates significantly from that baseline is flagged as potentially malicious. + +This is fundamentally different from signature-based detection (like traditional WAFs), which compares requests against a database of known attack patterns. Signature detection can only catch attacks it already knows about. Anomaly detection can catch attacks it has never seen before, as long as they look different from normal traffic. + +### Why It Matters + +In December 2021, the Log4Shell vulnerability (CVE-2021-44228) was disclosed. Within hours, attackers were sending requests containing `${jndi:ldap://...}` payloads to millions of servers. WAF vendors scrambled to push rule updates, but many organizations were exposed for days before rules were available. An anomaly detection system trained on normal traffic would have flagged these requests immediately. The `${jndi:` substring has high entropy, unusual character distribution, and doesn't match any legitimate path or query pattern. + +The 2020 SolarWinds attack (SUNBURST) is another example. The attackers used legitimate-looking HTTP requests to blend in with normal Orion platform traffic. But their request timing, path patterns, and behavioral fingerprints were subtly different from real management traffic. A well-tuned anomaly detector monitoring those signals could have raised early warnings. + +### How It Works + +AngelusVigil uses an **autoencoder** for anomaly detection. The concept is simple: train a neural network to compress and reconstruct normal traffic. If a new request can't be reconstructed well (high reconstruction error), it's probably abnormal. + +``` +Input (35 features) ──► Encoder ──► Bottleneck (6 dims) ──► Decoder ──► Reconstruction (35 features) + │ + Compare with input + │ + MSE = anomaly score +``` + +The autoencoder architecture in this project: + +``` +35 ──► 24 ──► 12 ──► 6 ──► 12 ──► 24 ──► 35 + input bottleneck output +``` + +The bottleneck forces the network to learn a compressed representation of normal traffic. Attack requests contain patterns the autoencoder never saw during training, so they reconstruct poorly. The reconstruction error (mean squared error between input and output) becomes the anomaly score. + +During training, only normal (benign) traffic is used. The model learns to compress and reconstruct legitimate requests. During inference, the reconstruction error is compared against a calibrated threshold (set at the 99.5th percentile of validation set errors). Requests exceeding this threshold are flagged as anomalous. + +### Common Attacks That Anomaly Detection Catches + +1. **Zero-day exploits** - Novel attack payloads that don't match any known signatures. The Log4Shell example above is a perfect case: the request structure is so unusual that anomaly detection flags it even without a specific rule. + +2. **Slow-and-low attacks** - Attackers who spread requests across time to evade rate limits. The windowed aggregation features (unique paths, method entropy, inter-request timing) capture behavioral patterns that per-request rules miss. + +3. **Application-layer DDoS** - Requests that are individually valid but collectively abnormal. A legitimate user doesn't hit 200 unique paths in 5 minutes with perfectly uniform 100ms spacing between requests. The per-IP windowed features expose this. + +4. **Credential stuffing** - Repeated POST requests to login endpoints from a single IP or a distributed botnet. Per-IP features like `req_count_1m`, `error_rate_5m`, and `unique_paths_5m` build a behavioral profile that catches this even when individual requests look normal. + +### Limitations + +Anomaly detection produces false positives. A legitimate user doing something unusual (bulk API calls, automated testing) will trigger alerts. That's why AngelusVigil uses an ensemble approach rather than relying on the autoencoder alone, and why it supports analyst review labels for active learning. + +It also can't tell you *what* an attack is. The autoencoder says "this is weird" but doesn't say "this is SQL injection." That's what the rule engine and random forest classifier add. + +## Ensemble Learning + +### What It Is + +Ensemble learning combines predictions from multiple models to make a final decision. The idea is that different models have different strengths and weaknesses. Combining them produces better results than any single model alone. + +AngelusVigil uses three models with weighted voting: + +``` + ┌──────────────────┐ + │ Autoencoder (AE) │ Weight: 0.40 + │ Unsupervised │ "How weird is this request?" + └────────┬─────────┘ + │ +┌──────────┐ ┌───────────┴───────────┐ ┌──────────────┐ +│ Feature │───►│ Ensemble Fusion │───►│ Final Score │ +│ Vector │ │ Weighted Average │ │ [0.0, 1.0] │ +└──────────┘ └───────────┬───────────┘ └──────────────┘ + │ + ┌────────┴─────────┐ + │ Random Forest │ Weight: 0.40 + │ Supervised │ "Is this an attack?" + └────────┬─────────┘ + │ + ┌────────┴─────────┐ + │ Isolation Forest │ Weight: 0.20 + │ Unsupervised │ "Is this an outlier?" + └──────────────────┘ +``` + +### Why Three Models? + +Each model brings a different perspective: + +**Autoencoder (40% weight)** - Trained only on normal traffic. Good at catching anything that looks "off" compared to the baseline. High recall (catches most attacks) but can be noisy (flags unusual but benign requests too). The reconstruction error approach means it doesn't need labeled attack data to learn. + +**Random Forest (40% weight)** - Trained on labeled data (both normal and attack samples). Good at classifying known attack types with high precision. Uses SMOTE oversampling to handle the class imbalance problem (attacks are rare compared to normal traffic). Can't catch attack types it wasn't trained on. + +**Isolation Forest (20% weight)** - Another unsupervised approach that isolates anomalies by random partitioning. Anomalies require fewer splits to isolate because they sit in sparse regions of the feature space. Gets lower weight because it overlaps somewhat with the autoencoder's anomaly detection capability but uses a fundamentally different algorithm (tree-based vs. neural network). + +### Why It Matters + +In 2014, JPMorgan Chase was breached through a compromised web server. Their IDS flagged the initial access, but a single model's alert was classified as a false positive and ignored. The attackers maintained access for two months, eventually compromising 76 million household records. Ensemble systems make this kind of single-model failure less likely because multiple independent models would need to agree that traffic is benign. + +### How Scores Are Combined + +The ensemble fusion happens in two stages. First, raw model scores are normalized to [0, 1]: + +```python +def normalize_ae_score(error: float, threshold: float) -> float: + if threshold <= 0: + return 0.0 + return min(error / (threshold * 2), 1.0) + +def normalize_if_score(raw_score: float) -> float: + return (1 - raw_score) / 2.0 +``` + +The autoencoder score is normalized against 2x the calibrated threshold. The isolation forest score is inverted (sklearn returns negative values for anomalies, positive for normal) and scaled. + +Then the normalized scores are fused with weighted averaging: + +```python +def fuse_scores(scores: dict[str, float], weights: dict[str, float]) -> float: + total = 0.0 + weight_sum = 0.0 + for key, weight in weights.items(): + if key in scores: + total += scores[key] * weight + weight_sum += weight + if weight_sum == 0: + return 0.0 + return total / weight_sum +``` + +The fusion function gracefully handles missing models. If only the autoencoder and random forest are loaded, the isolation forest weight is excluded and the remaining weights are renormalized. This supports partial model availability during retraining. + +Finally, the ML ensemble score is blended with the rule engine score: + +```python +def blend_scores(ml_score: float, rule_score: float, ml_weight: float = 0.7) -> float: + return min(ml_score * ml_weight + rule_score * (1.0 - ml_weight), 1.0) +``` + +ML gets 70% influence, rules get 30%. This means a high-confidence rule match (like a Log4Shell pattern) still contributes significantly to the final score even if the ML models are uncertain. + +### Severity Classification + +The blended score maps to three severity tiers: + +| Score Range | Severity | Action | +|---|---|---| +| 0.70 - 1.00 | HIGH | Store + publish alert + WebSocket broadcast | +| 0.50 - 0.69 | MEDIUM | Store + publish alert | +| 0.00 - 0.49 | LOW | Log only | + +## Feature Engineering for HTTP Traffic + +### What It Is + +Feature engineering transforms raw data (nginx log lines) into numeric vectors that ML models can process. The quality of your features determines the ceiling of your model's performance. You can always improve a model's architecture, but you can't overcome bad features. + +### Why It Matters + +A raw nginx log line looks like this: + +``` +192.168.1.100 - - [15/Mar/2026:14:22:31 +0000] "GET /api/users?id=1' OR '1'='1 HTTP/1.1" 200 1234 "-" "sqlmap/1.5" +``` + +A human can immediately spot the SQL injection payload and the sqlmap user agent. But an ML model needs numbers. Feature engineering is the bridge between "text a human can read" and "numbers a model can learn from." + +### The 35-Dimensional Feature Vector + +AngelusVigil extracts two types of features that are concatenated into a single 35-dimensional vector: + +**23 Per-Request Features** (stateless, computed from a single log entry): + +| Feature | Type | What It Captures | +|---|---|---| +| `http_method` | categorical | GET, POST, PUT, DELETE, etc. | +| `path_depth` | numeric | Number of `/` segments (deeper paths are unusual) | +| `path_entropy` | numeric | Shannon entropy of the path string (attack payloads have high entropy) | +| `path_length` | numeric | Raw character count | +| `query_string_length` | numeric | Long query strings often carry injection payloads | +| `query_param_count` | numeric | Number of `&`-separated parameters | +| `has_encoded_chars` | boolean | URL-encoded characters like `%27` (single quote) | +| `has_double_encoding` | boolean | Double-encoded chars like `%2527` (evasion technique) | +| `status_code` | numeric | HTTP response code | +| `status_class` | categorical | 2xx, 3xx, 4xx, 5xx grouping | +| `response_size` | numeric | Body size in bytes | +| `hour_of_day` | numeric | 0-23, captures time-of-day patterns | +| `day_of_week` | numeric | 0-6, captures weekly patterns | +| `is_weekend` | boolean | Weekend vs weekday | +| `ua_length` | numeric | User-Agent string length | +| `ua_entropy` | numeric | Shannon entropy of the UA string | +| `is_known_bot` | boolean | Matches known bot signatures (Googlebot, etc.) | +| `is_known_scanner` | boolean | Matches scanner signatures (sqlmap, nikto, etc.) | +| `has_attack_pattern` | boolean | Matches combined attack regex | +| `special_char_ratio` | numeric | Ratio of non-alphanumeric characters in the path | +| `file_extension` | categorical | `.php`, `.asp`, `.env`, etc. | +| `country_code` | categorical | GeoIP country code | +| `is_private_ip` | boolean | RFC 1918 private address | + +**12 Per-IP Windowed Features** (stateful, computed from Redis sorted sets): + +| Feature | Window | What It Captures | +|---|---|---| +| `req_count_1m` | 1 min | Immediate request rate (burst detection) | +| `req_count_5m` | 5 min | Short-term sustained rate | +| `req_count_10m` | 10 min | Medium-term pattern | +| `error_rate_5m` | 5 min | Ratio of 4xx/5xx responses (scanners trigger lots of errors) | +| `unique_paths_5m` | 5 min | Path diversity (scanners enumerate many paths) | +| `unique_uas_10m` | 10 min | User-Agent diversity (rotating UAs is a bot fingerprint) | +| `method_entropy_5m` | 5 min | Shannon entropy of HTTP methods used | +| `avg_response_size_5m` | 5 min | Mean response body size | +| `status_diversity_5m` | 5 min | Count of distinct status codes | +| `path_depth_variance_5m` | 5 min | Variance in path depth | +| `inter_request_time_mean` | 10 min | Average time between requests in ms | +| `inter_request_time_std` | 10 min | Standard deviation of request intervals | + +### Shannon Entropy + +Shannon entropy measures the randomness of a string. The formula: + +``` +H(s) = -Σ (count(c) / len(s)) * log2(count(c) / len(s)) +``` + +Normal paths like `/api/v1/users` have low entropy because characters repeat and follow predictable patterns. Attack payloads like `/api/search?q=%27%20UNION%20SELECT%20*%20FROM%20users--` have high entropy because they contain a wide variety of characters. + +The implementation in the codebase: + +```python +def _shannon_entropy(s: str) -> float: + if not s: + return 0.0 + length = len(s) + counts = Counter(s) + return -sum((c / length) * math.log2(c / length) for c in counts.values()) +``` + +This is applied to both the request path and the User-Agent string. Legitimate User-Agents like `Mozilla/5.0 (Windows NT 10.0; Win64; x64)` have predictable entropy ranges. Scanner tools like `sqlmap/1.5.2#stable` or randomized UAs used by botnets tend to fall outside those ranges. + +### Why Windowed Features Need Redis + +Per-request features are stateless. You can compute them from a single log line. But behavioral detection requires context: "Is this IP sending requests faster than usual? Has it hit an unusual number of distinct paths?" + +AngelusVigil stores per-IP request history in Redis sorted sets, with the Unix timestamp as the score. This makes windowed queries efficient: + +``` +ZADD ip:192.168.1.100:requests +ZCOUNT ip:192.168.1.100:requests <5m_ago> +inf +``` + +The `ZCOUNT` operation returns how many requests from this IP fell within the last 5 minutes, computed in O(log N) time. Seven sorted sets per IP track different dimensions (requests, paths, statuses, user agents, sizes, methods, depths), and all writes + reads happen in a single pipelined round-trip to minimize latency. + +Keys auto-expire after 15 minutes (`KEY_TTL = 900`), so Redis memory usage stays bounded even under heavy traffic. + +## Rule-Based Detection (Cold-Start) + +### What It Is + +Rule-based detection uses handcrafted patterns to identify known attack types. AngelusVigil's rule engine is inspired by the ModSecurity Core Rule Set (CRS), the open source WAF ruleset used by OWASP. + +### Why It Matters + +ML models need training data. On day one, you have no data and no models. The rule engine provides immediate protection with zero training, covering the most common web attack categories. + +Even after ML models are trained, rules remain valuable. They catch well-known attacks with high confidence and provide explainability ("this request matched the SQL injection pattern") that ML scores alone can't. + +### Attack Pattern Rules + +The rule engine evaluates 9 regex-based patterns against each request URI: + +| Rule | Score | What It Catches | +|---|---|---| +| LOG4SHELL | 0.95 | `${jndi:ldap://...}` and variations (CVE-2021-44228) | +| COMMAND_INJECTION | 0.90 | Shell metacharacters, backtick execution, pipe chains | +| SQL_INJECTION | 0.85 | `UNION SELECT`, `OR 1=1`, comment sequences, `SLEEP()` | +| XSS | 0.80 | `