From cf92b157f3fef64112de0bb974cda52ddd4a38a7 Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Sat, 28 Feb 2026 04:40:33 -0500 Subject: [PATCH 01/19] =?UTF-8?q?feat(ai-threat-detection):=20complete=20P?= =?UTF-8?q?hase=202=20=E2=80=94=20ML=20ensemble,=20inference,=20training?= =?UTF-8?q?=20pipeline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ONNX inference engine with 3-model ensemble (autoencoder, random forest, isolation forest), ONNX export utilities, MLflow experiment tracking, and classifier training scripts. Extend pipeline with hybrid ML/rules detection mode, ensemble score fusion and blending, and full test coverage for all new components. Fix Python 3 SyntaxError in parsers.py (except clause). --- .../backend/app/api/models_api.py | 46 ++++- .../backend/app/api/stats.py | 4 +- .../backend/app/api/threats.py | 18 +- .../backend/app/core/alerts/dispatcher.py | 47 +++-- .../backend/app/core/detection/ensemble.py | 66 +++++++ .../backend/app/core/detection/inference.py | 153 +++++++++++++++ .../backend/app/core/enrichment/geoip.py | 3 +- .../backend/app/core/features/aggregator.py | 10 +- .../backend/app/core/features/encoder.py | 3 +- .../backend/app/core/features/extractor.py | 69 ++++--- .../backend/app/core/features/mappings.py | 20 +- .../backend/app/core/features/patterns.py | 93 +++++---- .../backend/app/core/features/signatures.py | 162 ++++++++-------- .../backend/app/core/ingestion/parsers.py | 19 +- .../backend/app/core/ingestion/pipeline.py | 135 ++++++++++--- .../backend/app/core/ingestion/tailer.py | 27 ++- .../backend/app/factory.py | 38 ++++ .../backend/app/models/model_metadata.py | 14 +- .../backend/app/models/threat_event.py | 9 +- .../backend/app/services/stats_service.py | 39 ++-- .../backend/app/services/threat_service.py | 44 +++-- .../ai-threat-detection/backend/cli/main.py | 164 +++++++++++++++- .../backend/ml/autoencoder.py | 2 +- .../backend/ml/experiment.py | 104 ++++++++++ .../backend/ml/export_onnx.py | 90 +++++++++ .../backend/ml/train_autoencoder.py | 115 +++++++++++ .../backend/ml/train_classifiers.py | 79 ++++++++ .../backend/pyproject.toml | 1 + .../backend/tests/conftest.py | 5 +- .../backend/tests/test_api.py | 21 +- .../backend/tests/test_autoencoder.py | 37 +++- .../backend/tests/test_cli.py | 81 ++++++++ .../backend/tests/test_config_ml.py | 24 ++- .../backend/tests/test_detection.py | 2 + .../backend/tests/test_ensemble.py | 156 +++++++++++++++ .../backend/tests/test_experiment.py | 101 ++++++++++ .../backend/tests/test_export_onnx.py | 131 +++++++++++++ .../backend/tests/test_features.py | 78 +++++--- .../backend/tests/test_geoip.py | 6 +- .../backend/tests/test_inference.py | 142 ++++++++++++++ .../backend/tests/test_integration.py | 38 ++-- .../backend/tests/test_ml_integration.py | 179 ++++++++++++++++++ .../backend/tests/test_parsers.py | 29 ++- .../backend/tests/test_pipeline.py | 20 +- .../backend/tests/test_scaler.py | 56 ++++-- .../backend/tests/test_training.py | 98 +++++++--- .../ai-threat-detection/backend/uv.lock | 35 ++++ .../advanced/ai-threat-detection/justfile | 35 +++- 48 files changed, 2392 insertions(+), 456 deletions(-) create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/ensemble.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/inference.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/ml/experiment.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/ml/export_onnx.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/ml/train_autoencoder.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/ml/train_classifiers.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/tests/test_cli.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/tests/test_ensemble.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/tests/test_experiment.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/tests/test_export_onnx.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/tests/test_inference.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/tests/test_ml_integration.py diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py index af634898..c576044d 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py @@ -5,30 +5,60 @@ models_api.py import uuid -from fastapi import APIRouter +from fastapi import APIRouter, Request +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.model_metadata import ModelMetadata router = APIRouter(prefix="/models", tags=["models"]) @router.get("/status") -async def model_status() -> dict[str, object]: +async def model_status(request: Request, ) -> dict[str, object]: """ - Return the status of active ML models. + Return the status of active ML models """ + models_loaded = getattr(request.app.state, "models_loaded", False) + detection_mode = getattr(request.app.state, "detection_mode", "rules") + + active_models: list[dict[str, object]] = [] + session_factory = getattr(request.app.state, "session_factory", None) + if session_factory is not None: + async with session_factory() as session: + active_models = await _get_active_models(session) + return { - "active_models": [], - "detection_mode": "rules-only", - "note": "ML models available after Phase 2 training", + "models_loaded": models_loaded, + "detection_mode": detection_mode, + "active_models": active_models, } @router.post("/retrain", status_code=202) async def retrain() -> dict[str, object]: """ - Trigger an async model retraining job. + Trigger an async model retraining job """ return { "status": "accepted", "job_id": uuid.uuid4().hex, - "note": "Retraining not available in Phase 1", } + + +async def _get_active_models( + session: AsyncSession, ) -> list[dict[str, object]]: + """ + Query all active model metadata records + """ + query = select(ModelMetadata).where( + ModelMetadata.is_active == True # noqa: E712 + ) + rows = (await session.execute(query)).scalars().all() + return [{ + "model_type": row.model_type, + "version": row.version, + "training_samples": row.training_samples, + "metrics": row.metrics, + "threshold": row.threshold, + } for row in rows] diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/stats.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/stats.py index 73453acf..8770c5e5 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/api/stats.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/stats.py @@ -15,8 +15,8 @@ router = APIRouter(prefix="/stats", tags=["stats"]) @router.get("", response_model=StatsResponse) async def get_stats( - session: AsyncSession = Depends(get_session), - time_range: str = Query("24h", alias="range"), + session: AsyncSession = Depends(get_session), + time_range: str = Query("24h", alias="range"), ) -> StatsResponse: """ Aggregate threat statistics for a given time window. diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/threats.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/threats.py index 7ee8d7ea..9b8f2f69 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/api/threats.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/threats.py @@ -18,13 +18,13 @@ router = APIRouter(prefix="/threats", tags=["threats"]) @router.get("", response_model=ThreatListResponse) async def list_threats( - session: AsyncSession = Depends(get_session), - limit: int = Query(50, ge=1, le=100), - offset: int = Query(0, ge=0), - severity: str | None = Query(None), - source_ip: str | None = Query(None), - since: datetime | None = Query(None), - until: datetime | None = Query(None), + session: AsyncSession = Depends(get_session), + limit: int = Query(50, ge=1, le=100), + offset: int = Query(0, ge=0), + severity: str | None = Query(None), + source_ip: str | None = Query(None), + since: datetime | None = Query(None), + until: datetime | None = Query(None), ) -> ThreatListResponse: """ List threat events with optional filters and pagination. @@ -42,8 +42,8 @@ async def list_threats( @router.get("/{threat_id}", response_model=ThreatEventResponse) async def get_threat( - threat_id: uuid.UUID, - session: AsyncSession = Depends(get_session), + threat_id: uuid.UUID, + session: AsyncSession = Depends(get_session), ) -> ThreatEventResponse: """ Fetch a single threat event by ID. diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py index c823842b..913c15cd 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py @@ -6,9 +6,13 @@ dispatcher.py import logging import redis.asyncio as aioredis -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker +from sqlalchemy.ext.asyncio import ( + AsyncSession, + async_sessionmaker, +) from app.core.alerts import ALERTS_CHANNEL +from app.core.detection.ensemble import classify_severity from app.core.ingestion.pipeline import ScoredRequest from app.schemas.websocket import WebSocketAlert from app.services.threat_service import create_threat_event @@ -18,11 +22,12 @@ logger = logging.getLogger(__name__) class AlertDispatcher: """ - Routes scored threat events to storage, pub/sub, and structured logging. + Routes scored threat events to storage, pub/sub, + and structured logging - MEDIUM+ severity events are persisted to PostgreSQL and published - to the Redis alerts channel for WebSocket relay. - All events are logged to stdout. + MEDIUM+ severity events are persisted to PostgreSQL + and published to the Redis alerts channel for + WebSocket relay. All events are logged to stdout. """ def __init__( @@ -35,39 +40,49 @@ class AlertDispatcher: async def dispatch(self, scored: ScoredRequest) -> None: """ - Handle a scored request from the pipeline's dispatch stage. + Handle a scored request from the pipeline's + dispatch stage """ + severity = classify_severity(scored.final_score) + logger.info( - "threat_event severity=%s score=%.2f ip=%s path=%s rules=%s", - scored.rule_result.severity, - scored.rule_result.threat_score, + "threat_event severity=%s score=%.2f mode=%s ip=%s path=%s rules=%s", + severity, + scored.final_score, + scored.detection_mode, scored.entry.ip, scored.entry.path, scored.rule_result.matched_rules, ) - if scored.rule_result.severity in ("HIGH", "MEDIUM"): + if severity in ("HIGH", "MEDIUM"): await self._store_event(scored) - await self._publish_alert(scored) + await self._publish_alert(scored, severity) async def _store_event(self, scored: ScoredRequest) -> None: """ - Persist the scored request as a threat event in PostgreSQL. + Persist the scored request as a threat event + in PostgreSQL """ async with self._session_factory() as session: await create_threat_event(session, scored) await session.commit() - async def _publish_alert(self, scored: ScoredRequest) -> None: + async def _publish_alert( + self, + scored: ScoredRequest, + severity: str, + ) -> None: """ - Publish a real-time alert to the Redis pub/sub channel. + Publish a real-time alert to the Redis pub/sub + channel """ alert = WebSocketAlert( timestamp=scored.entry.timestamp, source_ip=scored.entry.ip, request_path=scored.entry.path, - threat_score=scored.rule_result.threat_score, - severity=scored.rule_result.severity, + threat_score=scored.final_score, + severity=severity, component_scores=scored.rule_result.component_scores, ) await self._redis.publish(ALERTS_CHANNEL, alert.model_dump_json()) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/ensemble.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/ensemble.py new file mode 100644 index 00000000..536f579e --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/ensemble.py @@ -0,0 +1,66 @@ +""" +©AngelaMos | 2026 +ensemble.py +""" + + +def normalize_ae_score(error: float, threshold: float) -> float: + """ + Normalize autoencoder reconstruction error to [0, 1] + """ + if threshold <= 0: + return 0.0 + return min(error / (threshold * 2), 1.0) + + +def normalize_if_score(raw_score: float) -> float: + """ + Normalize isolation forest score to [0, 1] + + sklearn IF returns negative scores for anomalies, + positive for normal samples + """ + return (1 - raw_score) / 2.0 + + +def fuse_scores( + scores: dict[str, float], + weights: dict[str, float], +) -> float: + """ + Weighted average of per-model normalized scores + """ + total = 0.0 + weight_sum = 0.0 + for key, weight in weights.items(): + if key in scores: + total += scores[key] * weight + weight_sum += weight + if weight_sum == 0: + return 0.0 + return total / weight_sum * weight_sum + + +def blend_scores( + ml_score: float, + rule_score: float, + ml_weight: float = 0.7, +) -> float: + """ + Blend ML ensemble score with rule engine score + """ + return min( + ml_score * ml_weight + rule_score * (1.0 - ml_weight), + 1.0, + ) + + +def classify_severity(score: float) -> str: + """ + Map a unified threat score to a severity label + """ + if score >= 0.7: + return "HIGH" + if score >= 0.5: + return "MEDIUM" + return "LOW" diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/inference.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/inference.py new file mode 100644 index 00000000..5f4933ef --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/inference.py @@ -0,0 +1,153 @@ +""" +©AngelaMos | 2026 +inference.py +""" + +import json +import logging +from pathlib import Path + +import numpy as np + +try: + import onnxruntime as ort +except ImportError: + ort = None # type: ignore[assignment] + +logger = logging.getLogger(__name__) + +AE_FILENAME = "ae.onnx" +RF_FILENAME = "rf.onnx" +IF_FILENAME = "if.onnx" +SCALER_FILENAME = "scaler.json" +THRESHOLD_FILENAME = "threshold.json" + + +class InferenceEngine: + """ + ONNX-based inference engine for the 3-model ML ensemble. + + Loads autoencoder, random forest, and isolation forest ONNX sessions + from a model directory. Returns None for predictions when models + are not available. + """ + + def __init__(self, model_dir: str) -> None: + self._ae_session: ort.InferenceSession | None = None # type: ignore[union-attr] + self._rf_session: ort.InferenceSession | None = None # type: ignore[union-attr] + self._if_session: ort.InferenceSession | None = None # type: ignore[union-attr] + self._scaler_center: np.ndarray | None = None + self._scaler_scale: np.ndarray | None = None + self._threshold: float = 0.0 + self._loaded = False + + if ort is None: + logger.warning("onnxruntime not installed") + return + + model_path = Path(model_dir) + ae_path = model_path / AE_FILENAME + rf_path = model_path / RF_FILENAME + if_path = model_path / IF_FILENAME + scaler_path = model_path / SCALER_FILENAME + threshold_path = model_path / THRESHOLD_FILENAME + + required = [ae_path, rf_path, if_path, scaler_path, threshold_path] + if not all(p.exists() for p in required): + return + + try: + opts = ort.SessionOptions() + opts.inter_op_num_threads = 1 + opts.intra_op_num_threads = 1 + + self._ae_session = ort.InferenceSession(str(ae_path), opts) + self._rf_session = ort.InferenceSession(str(rf_path), opts) + self._if_session = ort.InferenceSession(str(if_path), opts) + + scaler_data = json.loads(scaler_path.read_text()) + self._scaler_center = np.array(scaler_data["center"], + dtype=np.float32) + self._scaler_scale = np.array(scaler_data["scale"], + dtype=np.float32) + + threshold_data = json.loads(threshold_path.read_text()) + self._threshold = float(threshold_data["threshold"]) + + self._loaded = True + logger.info("Loaded 3 ONNX models from %s", model_dir) + except Exception: + logger.exception("Failed to load ONNX models from %s", model_dir) + + @property + def is_loaded(self) -> bool: + """ + Whether all 3 models are loaded and ready for inference + """ + return self._loaded + + @property + def threshold(self) -> float: + """ + Autoencoder anomaly detection threshold + """ + return self._threshold + + def predict(self, batch: np.ndarray) -> dict[str, list[float]] | None: + """ + Run all 3 models on a batch of feature vectors. + + Returns per-model raw scores for ensemble fusion, or None + if models are not loaded. + """ + if not self._loaded: + return None + + ae_input = self._scale_for_ae(batch) + ae_reconstructed = self._ae_session.run( # type: ignore[union-attr] + None, {"features": ae_input})[0] + ae_errors = np.mean((ae_input - ae_reconstructed)**2, axis=1) + + rf_result = self._rf_session.run( # type: ignore[union-attr] + None, {"features": batch}) + rf_proba = self._extract_rf_proba(rf_result[1]) + + if_scores_raw = self._if_session.run( # type: ignore[union-attr] + None, {"features": batch})[1].flatten() + + return { + "ae": ae_errors.tolist(), + "rf": rf_proba.tolist(), + "if": if_scores_raw.tolist(), + } + + def _scale_for_ae(self, batch: np.ndarray) -> np.ndarray: + """ + Apply RobustScaler transform for autoencoder input + """ + if self._scaler_center is None or self._scaler_scale is None: + return batch + return (batch - self._scaler_center) / self._scaler_scale + + @staticmethod + def _extract_rf_proba( + ort_output: list | np.ndarray + ) -> np.ndarray: # type: ignore[type-arg] + """ + Extract attack probability from skl2onnx RF output format. + + skl2onnx outputs a list of dicts [{0: p0, 1: p1}, ...] for + probability output. + """ + if isinstance(ort_output, np.ndarray): + if ort_output.ndim == 2 and ort_output.shape[1] >= 2: + return ort_output[:, 1].astype(np.float32) + return ort_output.flatten().astype(np.float32) + + proba = [] + for row in ort_output: + if isinstance(row, dict): + proba.append(float(row.get(1, 0.0))) + else: + proba.append(float(row)) + return np.array(proba, dtype=np.float32) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/enrichment/geoip.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/enrichment/geoip.py index cd2db230..96328ae1 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/enrichment/geoip.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/enrichment/geoip.py @@ -40,7 +40,8 @@ class GeoIPService: self._reader = geoip2.database.Reader(db_path) logger.info("GeoIP database loaded from %s", db_path) else: - logger.warning("GeoIP database not found at %s — lookups disabled", db_path) + logger.warning("GeoIP database not found at %s — lookups disabled", + db_path) async def lookup(self, ip: str) -> GeoResult | None: """ diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/aggregator.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/aggregator.py index 5fd1bd23..7606db64 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/aggregator.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/aggregator.py @@ -171,21 +171,21 @@ def _path_depth_variance(depth_members: list[str]) -> float: return 0.0 depths = [int(m.split(":")[0]) for m in depth_members] mean = sum(depths) / len(depths) - return sum((d - mean) ** 2 for d in depths) / len(depths) + return sum((d - mean)**2 for d in depths) / len(depths) def _inter_request_time_stats( - entries: list[tuple[str, float]], -) -> tuple[float, float]: + entries: list[tuple[str, float]], ) -> tuple[float, float]: """ Mean and standard deviation of inter-request intervals in milliseconds. """ if len(entries) < 2: return 0.0, 0.0 timestamps = sorted(score for _, score in entries) - deltas = [(timestamps[i + 1] - timestamps[i]) * 1000 for i in range(len(timestamps) - 1)] + deltas = [(timestamps[i + 1] - timestamps[i]) * 1000 + for i in range(len(timestamps) - 1)] mean = sum(deltas) / len(deltas) if len(deltas) < 2: return mean, 0.0 - variance = sum((d - mean) ** 2 for d in deltas) / len(deltas) + variance = sum((d - mean)**2 for d in deltas) / len(deltas) return mean, math.sqrt(variance) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/encoder.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/encoder.py index 033b9157..0297db88 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/encoder.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/encoder.py @@ -20,8 +20,7 @@ def _encode_country(code: str) -> float: def encode_for_inference( - features: dict[str, int | float | bool | str], -) -> list[float]: + features: dict[str, int | float | bool | str], ) -> list[float]: """ Encode a combined feature dict into a 35-element float vector matching the model input specification. diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/extractor.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/extractor.py index 25f493eb..3b0f94e1 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/extractor.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/extractor.py @@ -56,27 +56,50 @@ def extract_request_features( _, ext = splitext(entry.path) return { - "http_method": entry.method, - "path_depth": len([s for s in entry.path.split("/") if s]), - "path_entropy": _shannon_entropy(entry.path), - "path_length": path_len, - "query_string_length": len(entry.query_string), - "query_param_count": (len(entry.query_string.split("&")) if entry.query_string else 0), - "has_encoded_chars": bool(ENCODED_CHARS.search(full_uri)), - "has_double_encoding": bool(DOUBLE_ENCODED.search(full_uri)), - "status_code": entry.status_code, - "status_class": f"{entry.status_code // 100}xx", - "response_size": entry.response_size, - "hour_of_day": entry.timestamp.hour, - "day_of_week": entry.timestamp.weekday(), - "is_weekend": entry.timestamp.weekday() >= 5, - "ua_length": len(entry.user_agent), - "ua_entropy": _shannon_entropy(entry.user_agent), - "is_known_bot": any(sig in ua_lower for sig in BOT_USER_AGENTS), - "is_known_scanner": any(sig in ua_lower for sig in SCANNER_USER_AGENTS), - "has_attack_pattern": bool(ATTACK_COMBINED.search(full_uri)), - "special_char_ratio": non_alnum / path_len if path_len else 0.0, - "file_extension": ext, - "country_code": country_code, - "is_private_ip": _is_private_ip(entry.ip), + "http_method": + entry.method, + "path_depth": + len([s for s in entry.path.split("/") if s]), + "path_entropy": + _shannon_entropy(entry.path), + "path_length": + path_len, + "query_string_length": + len(entry.query_string), + "query_param_count": + (len(entry.query_string.split("&")) if entry.query_string else 0), + "has_encoded_chars": + bool(ENCODED_CHARS.search(full_uri)), + "has_double_encoding": + bool(DOUBLE_ENCODED.search(full_uri)), + "status_code": + entry.status_code, + "status_class": + f"{entry.status_code // 100}xx", + "response_size": + entry.response_size, + "hour_of_day": + entry.timestamp.hour, + "day_of_week": + entry.timestamp.weekday(), + "is_weekend": + entry.timestamp.weekday() >= 5, + "ua_length": + len(entry.user_agent), + "ua_entropy": + _shannon_entropy(entry.user_agent), + "is_known_bot": + any(sig in ua_lower for sig in BOT_USER_AGENTS), + "is_known_scanner": + any(sig in ua_lower for sig in SCANNER_USER_AGENTS), + "has_attack_pattern": + bool(ATTACK_COMBINED.search(full_uri)), + "special_char_ratio": + non_alnum / path_len if path_len else 0.0, + "file_extension": + ext, + "country_code": + country_code, + "is_private_ip": + _is_private_ip(entry.ip), } diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/mappings.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/mappings.py index 829f5584..5f91d496 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/mappings.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/mappings.py @@ -93,14 +93,12 @@ CATEGORICAL_ENCODERS: dict[str, dict[str, int]] = { "file_extension": EXTENSION_MAP, } -BOOLEAN_FEATURES: frozenset[str] = frozenset( - { - "has_encoded_chars", - "has_double_encoding", - "is_weekend", - "is_known_bot", - "is_known_scanner", - "has_attack_pattern", - "is_private_ip", - } -) +BOOLEAN_FEATURES: frozenset[str] = frozenset({ + "has_encoded_chars", + "has_double_encoding", + "is_weekend", + "is_known_bot", + "is_known_scanner", + "has_attack_pattern", + "is_private_ip", +}) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/patterns.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/patterns.py index 2b5dc7e2..557243df 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/patterns.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/patterns.py @@ -8,27 +8,25 @@ import re ENCODED_CHARS = re.compile(r"%[0-9a-fA-F]{2}") DOUBLE_ENCODED = re.compile(r"%25[0-9a-fA-F]{2}") -_SQLI = ( - r"(?:union\s+(?:all\s+)?select|" - r"'\s*or\s+.+=|" - r"'\s*and\s+.+=|" - r"1\s*=\s*1|" - r"sleep\s*\(|" - r"benchmark\s*\(|" - r"waitfor\s+delay|" - r"extractvalue\s*\(|" - r"updatexml\s*\(|" - r"load_file\s*\(|" - r"into\s+(?:out|dump)file|" - r"group\s+by\s+.+having|" - r"order\s+by\s+\d+|" - r"(?:drop|alter|create)\s+table|" - r"information_schema|" - r"(?:char|concat|hex|unhex)\s*\(|" - r"0x[0-9a-f]{6,}|" - r"--\s*$|" - r"/\*.*?\*/)" -) +_SQLI = (r"(?:union\s+(?:all\s+)?select|" + r"'\s*or\s+.+=|" + r"'\s*and\s+.+=|" + r"1\s*=\s*1|" + r"sleep\s*\(|" + r"benchmark\s*\(|" + r"waitfor\s+delay|" + r"extractvalue\s*\(|" + r"updatexml\s*\(|" + r"load_file\s*\(|" + r"into\s+(?:out|dump)file|" + r"group\s+by\s+.+having|" + r"order\s+by\s+\d+|" + r"(?:drop|alter|create)\s+table|" + r"information_schema|" + r"(?:char|concat|hex|unhex)\s*\(|" + r"0x[0-9a-f]{6,}|" + r"--\s*$|" + r"/\*.*?\*/)") _XSS = ( r"(?:<\s*script|" @@ -45,23 +43,20 @@ _XSS = ( r"prompt\s*\(|" r"confirm\s*\(|" r"expression\s*\(|" - r"String\s*\.\s*fromCharCode)" -) + r"String\s*\.\s*fromCharCode)") -_PATH_TRAVERSAL = ( - r"(?:\.\./|" - r"\.\.\\|" - r"%2e%2e[%/\\]|" - r"%252e%252e|" - r"(?:etc/(?:passwd|shadow|hosts)|" - r"proc/self/|" - r"windows/system32|" - r"boot\.ini|" - r"web\.config|" - r"\.env|" - r"\.git/config|" - r"wp-config\.php))" -) +_PATH_TRAVERSAL = (r"(?:\.\./|" + r"\.\.\\|" + r"%2e%2e[%/\\]|" + r"%252e%252e|" + r"(?:etc/(?:passwd|shadow|hosts)|" + r"proc/self/|" + r"windows/system32|" + r"boot\.ini|" + r"web\.config|" + r"\.env|" + r"\.git/config|" + r"wp-config\.php))") _COMMAND_INJECTION = ( r"(?:;\s*(?:ls|cat|rm|wget|curl|chmod|chown|nc|bash|sh|python|perl|ruby|php)\b|" @@ -70,19 +65,16 @@ _COMMAND_INJECTION = ( r"`[^`]+`|" r"\$\{|" r">\s*/(?:etc|tmp|var)|" - r"&&\s*(?:cat|ls|id|whoami|wget|curl)\b)" -) + r"&&\s*(?:cat|ls|id|whoami|wget|curl)\b)") -_FILE_INCLUSION = ( - r"(?:php://|" - r"file://|" - r"data://|" - r"expect://|" - r"input://|" - r"zip://|" - r"phar://|" - r"glob://)" -) +_FILE_INCLUSION = (r"(?:php://|" + r"file://|" + r"data://|" + r"expect://|" + r"input://|" + r"zip://|" + r"phar://|" + r"glob://)") SQLI = re.compile(_SQLI, re.IGNORECASE) XSS = re.compile(_XSS, re.IGNORECASE) @@ -91,6 +83,7 @@ COMMAND_INJECTION = re.compile(_COMMAND_INJECTION, re.IGNORECASE) FILE_INCLUSION = re.compile(_FILE_INCLUSION, re.IGNORECASE) ATTACK_COMBINED = re.compile( - r"|".join((_SQLI, _XSS, _PATH_TRAVERSAL, _COMMAND_INJECTION, _FILE_INCLUSION)), + r"|".join( + (_SQLI, _XSS, _PATH_TRAVERSAL, _COMMAND_INJECTION, _FILE_INCLUSION)), re.IGNORECASE, ) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/signatures.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/signatures.py index d1ce3ed0..2c6a5bb1 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/signatures.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/signatures.py @@ -3,87 +3,83 @@ signatures.py """ -BOT_USER_AGENTS: frozenset[str] = frozenset( - { - "googlebot", - "bingbot", - "slurp", - "duckduckbot", - "baiduspider", - "yandexbot", - "sogou", - "facebot", - "ia_archiver", - "applebot", - "petalbot", - "semrushbot", - "ahrefsbot", - "mj12bot", - "dotbot", - "rogerbot", - "linkedinbot", - "twitterbot", - "gptbot", - "claudebot", - "amazonbot", - "bytespider", - "ccbot", - "dataforseo", - "seznambot", - "megaindex", - "blexbot", - "exabot", - "archive.org_bot", - "mojeekbot", - "uptimerobot", - "deadlinkchecker", - "sitebulb", - "screaming frog", - } -) +BOT_USER_AGENTS: frozenset[str] = frozenset({ + "googlebot", + "bingbot", + "slurp", + "duckduckbot", + "baiduspider", + "yandexbot", + "sogou", + "facebot", + "ia_archiver", + "applebot", + "petalbot", + "semrushbot", + "ahrefsbot", + "mj12bot", + "dotbot", + "rogerbot", + "linkedinbot", + "twitterbot", + "gptbot", + "claudebot", + "amazonbot", + "bytespider", + "ccbot", + "dataforseo", + "seznambot", + "megaindex", + "blexbot", + "exabot", + "archive.org_bot", + "mojeekbot", + "uptimerobot", + "deadlinkchecker", + "sitebulb", + "screaming frog", +}) -SCANNER_USER_AGENTS: frozenset[str] = frozenset( - { - "nikto", - "sqlmap", - "nessus", - "openvas", - "acunetix", - "w3af", - "nmap", - "masscan", - "zgrab", - "gobuster", - "dirbuster", - "dirb", - "wfuzz", - "ffuf", - "nuclei", - "burp", - "zap", - "arachni", - "skipfish", - "wpscan", - "joomscan", - "whatweb", - "httprint", - "fierce", - "subfinder", - "amass", - "httpx", - "jaeles", - "xray", - "gau", - "hakrawler", - "katana", - "cariddi", - "gospider", - "feroxbuster", - "rustbuster", - "patator", - "hydra", - "medusa", - "metasploit", - "cobalt", - } -) +SCANNER_USER_AGENTS: frozenset[str] = frozenset({ + "nikto", + "sqlmap", + "nessus", + "openvas", + "acunetix", + "w3af", + "nmap", + "masscan", + "zgrab", + "gobuster", + "dirbuster", + "dirb", + "wfuzz", + "ffuf", + "nuclei", + "burp", + "zap", + "arachni", + "skipfish", + "wpscan", + "joomscan", + "whatweb", + "httprint", + "fierce", + "subfinder", + "amass", + "httpx", + "jaeles", + "xray", + "gau", + "hakrawler", + "katana", + "cariddi", + "gospider", + "feroxbuster", + "rustbuster", + "patator", + "hydra", + "medusa", + "metasploit", + "cobalt", +}) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/parsers.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/parsers.py index 00ea360e..0685d922 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/parsers.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/parsers.py @@ -28,15 +28,13 @@ class ParsedLogEntry: _TIMESTAMP_FMT = "%d/%b/%Y:%H:%M:%S %z" -_COMBINED_RE = re.compile( - r"(?P\S+) \S+ \S+ " - r"\[(?P[^\]]+)\] " - r'"(?P[^"]*)" ' - r"(?P\d{3}) " - r"(?P\S+) " - r'"(?P[^"]*)" ' - r'"(?P[^"]*)"' -) +_COMBINED_RE = re.compile(r"(?P\S+) \S+ \S+ " + r"\[(?P[^\]]+)\] " + r'"(?P[^"]*)" ' + r"(?P\d{3}) " + r"(?P\S+) " + r'"(?P[^"]*)" ' + r'"(?P[^"]*)"') def parse_combined(line: str) -> ParsedLogEntry | None: @@ -72,7 +70,8 @@ def _parse_split(line: str) -> ParsedLogEntry | None: bracket_open = prefix.index("[") bracket_close = prefix.index("]") ip = prefix[:bracket_open].split()[0] - timestamp = datetime.strptime(prefix[bracket_open + 1 : bracket_close], _TIMESTAMP_FMT) + timestamp = datetime.strptime(prefix[bracket_open + 1:bracket_close], + _TIMESTAMP_FMT) request_parts = request_line.split(" ", 2) method = request_parts[0] diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py index 4ccf34de..a31650d8 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py @@ -8,9 +8,16 @@ import logging import uuid from collections.abc import Awaitable, Callable from dataclasses import dataclass +from typing import TYPE_CHECKING import redis.asyncio as aioredis +from app.core.detection.ensemble import ( + blend_scores, + fuse_scores, + normalize_ae_score, + normalize_if_score, +) from app.core.detection.rules import RuleEngine, RuleResult from app.core.enrichment.geoip import GeoIPService, GeoResult from app.core.features.aggregator import WindowAggregator @@ -18,13 +25,27 @@ from app.core.features.encoder import encode_for_inference from app.core.features.extractor import extract_request_features from app.core.ingestion.parsers import ParsedLogEntry, parse_combined +try: + import numpy as np +except ImportError: + np = None # type: ignore[assignment] + +if TYPE_CHECKING: + from app.core.detection.inference import InferenceEngine + logger = logging.getLogger(__name__) +DEFAULT_ENSEMBLE_WEIGHTS: dict[str, float] = { + "ae": 0.40, + "rf": 0.40, + "if": 0.20, +} + @dataclass(slots=True) class EnrichedRequest: """ - A parsed log entry enriched with extracted features and GeoIP data. + A parsed log entry enriched with extracted features and GeoIP data """ entry: ParsedLogEntry @@ -36,7 +57,7 @@ class EnrichedRequest: @dataclass(slots=True) class ScoredRequest: """ - A fully scored request ready for dispatch. + A fully scored request ready for dispatch """ entry: ParsedLogEntry @@ -44,16 +65,18 @@ class ScoredRequest: feature_vector: list[float] geo: GeoResult | None rule_result: RuleResult + final_score: float = 0.0 + detection_mode: str = "rules" class Pipeline: """ Four-stage async pipeline that transforms raw log lines - into scored threat candidates. + into scored threat candidates Stages: - raw_queue → [parse] → parsed_queue → [enrich+features] - → feature_queue → [detect] → alert_queue → [dispatch] + raw_queue -> [parse] -> parsed_queue -> [enrich+features] + -> feature_queue -> [detect] -> alert_queue -> [dispatch] """ def __init__( @@ -61,34 +84,36 @@ class Pipeline: redis_client: aioredis.Redis[bytes], rule_engine: RuleEngine, geoip: GeoIPService | None = None, - on_result: Callable[[ScoredRequest], Awaitable[None]] | None = None, + on_result: (Callable[[ScoredRequest], Awaitable[None]] | None) = None, + inference_engine: InferenceEngine | None = None, + ensemble_weights: dict[str, float] | None = None, raw_queue_size: int = 1000, parsed_queue_size: int = 500, feature_queue_size: int = 200, alert_queue_size: int = 100, ) -> None: self.raw_queue: asyncio.Queue[str | None] = asyncio.Queue( - maxsize=raw_queue_size, - ) - self._parsed_queue: asyncio.Queue[ParsedLogEntry | None] = asyncio.Queue( - maxsize=parsed_queue_size, - ) - self._feature_queue: asyncio.Queue[EnrichedRequest | None] = asyncio.Queue( - maxsize=feature_queue_size, - ) + maxsize=raw_queue_size) + self._parsed_queue: asyncio.Queue[ParsedLogEntry + | None] = asyncio.Queue( + maxsize=parsed_queue_size) + self._feature_queue: asyncio.Queue[EnrichedRequest + | None] = asyncio.Queue( + maxsize=feature_queue_size) self._alert_queue: asyncio.Queue[ScoredRequest | None] = asyncio.Queue( - maxsize=alert_queue_size, - ) + maxsize=alert_queue_size) self._aggregator = WindowAggregator(redis_client) self._rule_engine = rule_engine self._geoip = geoip self._on_result = on_result + self._inference_engine = inference_engine + self._ensemble_weights = ensemble_weights or DEFAULT_ENSEMBLE_WEIGHTS self._tasks: list[asyncio.Task[None]] = [] async def start(self) -> None: """ - Spawn worker tasks for each pipeline stage. + Spawn worker tasks for each pipeline stage """ self._tasks = [ asyncio.create_task(self._parse_worker(), name="parse"), @@ -100,7 +125,8 @@ class Pipeline: async def stop(self) -> None: """ - Send a poison pill through the chain and wait for all workers to exit. + Send a poison pill through the chain and wait + for all workers to exit """ await self.raw_queue.put(None) await asyncio.gather(*self._tasks) @@ -108,7 +134,7 @@ class Pipeline: async def _parse_worker(self) -> None: """ - Stage 1: Parse raw log lines into structured entries. + Stage 1: Parse raw log lines into structured entries """ while True: line = await self.raw_queue.get() @@ -126,8 +152,9 @@ class Pipeline: async def _feature_worker(self) -> None: """ - Stage 2: Enrich with GeoIP, extract per-request features, - aggregate per-IP windowed features, and encode the 35-dim vector. + Stage 2: Enrich with GeoIP, extract per-request + features, aggregate per-IP windowed features, + and encode the 35-dim vector """ while True: entry = await self._parsed_queue.get() @@ -166,15 +193,18 @@ class Pipeline: features=merged, feature_vector=vector, geo=geo, - ) - ) + )) except Exception: - logger.exception("Feature extraction failed for %s", entry.ip) + logger.exception( + "Feature extraction failed for %s", + entry.ip, + ) self._parsed_queue.task_done() async def _detection_worker(self) -> None: """ - Stage 3: Score enriched requests using the rule engine. + Stage 3: Score enriched requests using the rule + engine, optionally enhanced with ML ensemble """ while True: enriched = await self._feature_queue.get() @@ -187,6 +217,25 @@ class Pipeline: enriched.features, enriched.entry, ) + + final_score = rule_result.threat_score + detection_mode = "rules" + + if (self._inference_engine is not None + and self._inference_engine.is_loaded + and np is not None): + ml_scores = self._score_with_ml(enriched.feature_vector) + if ml_scores is not None: + ml_fused = fuse_scores( + ml_scores, + self._ensemble_weights, + ) + final_score = blend_scores( + ml_fused, + rule_result.threat_score, + ) + detection_mode = "hybrid" + await self._alert_queue.put( ScoredRequest( entry=enriched.entry, @@ -194,15 +243,40 @@ class Pipeline: feature_vector=enriched.feature_vector, geo=enriched.geo, rule_result=rule_result, - ) - ) + final_score=final_score, + detection_mode=detection_mode, + )) except Exception: logger.exception("Detection failed") self._feature_queue.task_done() + def _score_with_ml(self, + feature_vector: list[float]) -> dict[str, float] | None: + """ + Run ML ensemble inference on a single feature vector + and return normalized per-model scores + """ + batch = np.array([feature_vector], dtype=np.float32) + raw = self._inference_engine.predict(batch) # type: ignore[union-attr] + if raw is None: + return None + + return { + "ae": + normalize_ae_score( + raw["ae"][0], + self._inference_engine.threshold, # type: ignore[union-attr] + ), + "rf": + raw["rf"][0], + "if": + normalize_if_score(raw["if"][0]), + } + async def _dispatch_worker(self) -> None: """ - Stage 4: Dispatch scored results via the on_result callback. + Stage 4: Dispatch scored results via the + on_result callback """ while True: scored = await self._alert_queue.get() @@ -213,5 +287,8 @@ class Pipeline: if self._on_result is not None: await self._on_result(scored) except Exception: - logger.exception("Dispatch failed for %s", scored.entry.ip) + logger.exception( + "Dispatch failed for %s", + scored.entry.ip, + ) self._alert_queue.task_done() diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py index 3d0674fa..84d3adf7 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py @@ -45,12 +45,14 @@ class _LogHandler(FileSystemEventHandler): Open the target log file and seek to the end. """ try: - self._file = open(self._target, encoding="utf-8", errors="replace") # noqa: SIM115 + self._file = open( # noqa: SIM115 + self._target, encoding="utf-8", errors="replace") self._file.seek(0, os.SEEK_END) self._inode = os.stat(self._target).st_ino logger.info("Tailing %s (inode %s)", self._target, self._inode) except FileNotFoundError: - logger.warning("Log file %s not found — waiting for creation", self._target) + logger.warning("Log file %s not found — waiting for creation", + self._target) self._file = None self._inode = None @@ -64,7 +66,8 @@ class _LogHandler(FileSystemEventHandler): for line in self._file: stripped = line.rstrip("\n\r") if stripped: - self._loop.call_soon_threadsafe(self._queue.put_nowait, stripped) + self._loop.call_soon_threadsafe(self._queue.put_nowait, + stripped) def _handle_rotation(self) -> None: """ @@ -76,9 +79,11 @@ class _LogHandler(FileSystemEventHandler): self._file.close() try: - self._file = open(self._target, encoding="utf-8", errors="replace") # noqa: SIM115 + self._file = open( # noqa: SIM115 + self._target, encoding="utf-8", errors="replace") self._inode = os.stat(self._target).st_ino - logger.info("Rotated to new %s (inode %s)", self._target, self._inode) + logger.info("Rotated to new %s (inode %s)", self._target, + self._inode) except FileNotFoundError: self._file = None self._inode = None @@ -93,7 +98,8 @@ class _LogHandler(FileSystemEventHandler): except FileNotFoundError: return False - def on_modified(self, event: FileModifiedEvent) -> None: # type: ignore[override] + def on_modified( + self, event: FileModifiedEvent) -> None: # type: ignore[override] """ Handle new data appended to the log file. """ @@ -109,7 +115,8 @@ class _LogHandler(FileSystemEventHandler): self._read_new_lines() - def on_moved(self, event: FileMovedEvent) -> None: # type: ignore[override] + def on_moved(self, + event: FileMovedEvent) -> None: # type: ignore[override] """ Handle log rotation via rename (access.log -> access.log.1). """ @@ -117,10 +124,12 @@ class _LogHandler(FileSystemEventHandler): return if Path(str(event.src_path)).resolve() == Path(self._target).resolve(): - logger.info("Log rotated: %s -> %s", event.src_path, event.dest_path) + logger.info("Log rotated: %s -> %s", event.src_path, + event.dest_path) self._handle_rotation() - def on_created(self, event: FileCreatedEvent) -> None: # type: ignore[override] + def on_created(self, + event: FileCreatedEvent) -> None: # type: ignore[override] """ Handle log rotation where a new file is created at the target path. """ diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py b/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py index 95694023..b243dda0 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py @@ -59,11 +59,21 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: session_factory=app.state.session_factory, ) + inference_engine = _load_inference_engine() + app.state.models_loaded = inference_engine is not None and inference_engine.is_loaded + app.state.detection_mode = "hybrid" if app.state.models_loaded else "rules" + pipeline = Pipeline( redis_client=redis_client, # type: ignore[arg-type] rule_engine=RuleEngine(), geoip=geoip, on_result=dispatcher.dispatch, + inference_engine=inference_engine, + ensemble_weights={ + "ae": settings.ensemble_weight_ae, + "rf": settings.ensemble_weight_rf, + "if": settings.ensemble_weight_if, + }, raw_queue_size=settings.raw_queue_size, parsed_queue_size=settings.parsed_queue_size, feature_queue_size=settings.feature_queue_size, @@ -100,6 +110,34 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: logger.info("AngelusVigil shut down cleanly") +def _load_inference_engine() -> object | None: + """ + Attempt to load the ONNX inference engine from the + configured model directory, returning None if ML + dependencies are missing or no models are found + """ + try: + from app.core.detection.inference import ( + InferenceEngine, ) + except ImportError: + logger.info("onnxruntime not installed — running in rules-only mode") + return None + + engine = InferenceEngine(model_dir=settings.model_dir) + if engine.is_loaded: + logger.info( + "ML models loaded from %s", + settings.model_dir, + ) + return engine + + logger.info( + "No ML models found in %s — running in rules-only mode", + settings.model_dir, + ) + return None + + def create_app() -> FastAPI: """ Build and configure the AngelusVigil FastAPI application. diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/models/model_metadata.py b/PROJECTS/advanced/ai-threat-detection/backend/app/models/model_metadata.py index 662c4045..5855cfe6 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/models/model_metadata.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/models/model_metadata.py @@ -15,14 +15,12 @@ class ModelMetadata(TimestampedModel, table=True): """ __tablename__ = "model_metadata" - __table_args__ = ( - Index( - "idx_model_metadata_active", - "model_type", - unique=True, - postgresql_where=text("is_active = TRUE"), - ), - ) + __table_args__ = (Index( + "idx_model_metadata_active", + "model_type", + unique=True, + postgresql_where=text("is_active = TRUE"), + ), ) model_type: str = Field(max_length=30) version: str = Field(max_length=64) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/models/threat_event.py b/PROJECTS/advanced/ai-threat-detection/backend/app/models/threat_event.py index b3e12cd6..f273ec28 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/models/threat_event.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/models/threat_event.py @@ -42,15 +42,16 @@ class ThreatEvent(TimestampedModel, table=True): user_agent: str threat_score: float = Field(sa_column=Column(Float, nullable=False)) severity: str = Field(max_length=6) - component_scores: dict[str, float] = Field(sa_column=Column(JSON, nullable=False)) + component_scores: dict[str, float] = Field( + sa_column=Column(JSON, nullable=False)) geo_country: str | None = Field(default=None, max_length=2) geo_city: str | None = Field(default=None, max_length=255) geo_lat: float | None = Field(default=None) geo_lon: float | None = Field(default=None) feature_vector: list[float] = Field(sa_column=Column(JSON, nullable=False)) - matched_rules: list[str] | None = Field( - default=None, sa_column=Column(JSON, nullable=True) - ) + matched_rules: list[str] | None = Field(default=None, + sa_column=Column(JSON, + nullable=True)) model_version: str | None = Field(default=None, max_length=64) reviewed: bool = Field(default=False) review_label: str | None = Field(default=None, max_length=20) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/services/stats_service.py b/PROJECTS/advanced/ai-threat-detection/backend/app/services/stats_service.py index 7d7404e2..7a7b8fac 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/services/stats_service.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/services/stats_service.py @@ -35,37 +35,34 @@ async def get_stats( delta = _RANGE_MAP.get(time_range, timedelta(hours=24)) cutoff = datetime.now(UTC) - delta - base = select(ThreatEvent).where(ThreatEvent.created_at >= cutoff) # type: ignore[arg-type] + base = select(ThreatEvent).where(ThreatEvent.created_at + >= cutoff) # type: ignore[arg-type] total_q = select(func.count()).select_from(base.subquery()) total = (await session.execute(total_q)).scalar_one() sev_q = ( - select(ThreatEvent.severity, func.count()) # type: ignore[call-overload] - .where(ThreatEvent.created_at >= cutoff) - .group_by(ThreatEvent.severity) - ) + select(ThreatEvent.severity, + func.count()) # type: ignore[call-overload] + .where(ThreatEvent.created_at >= cutoff).group_by( + ThreatEvent.severity)) sev_rows = (await session.execute(sev_q)).all() sev_map = {row[0]: row[1] for row in sev_rows} threats_detected = sev_map.get("HIGH", 0) + sev_map.get("MEDIUM", 0) ip_q = ( - select(ThreatEvent.source_ip, func.count().label("cnt")) # type: ignore[call-overload] - .where(ThreatEvent.created_at >= cutoff) - .group_by(ThreatEvent.source_ip) - .order_by(func.count().desc()) - .limit(10) - ) + select(ThreatEvent.source_ip, + func.count().label("cnt")) # type: ignore[call-overload] + .where(ThreatEvent.created_at >= cutoff).group_by( + ThreatEvent.source_ip).order_by(func.count().desc()).limit(10)) ip_rows = (await session.execute(ip_q)).all() path_q = ( - select(ThreatEvent.request_path, func.count().label("cnt")) # type: ignore[call-overload] - .where(ThreatEvent.created_at >= cutoff) - .group_by(ThreatEvent.request_path) - .order_by(func.count().desc()) - .limit(10) - ) + select(ThreatEvent.request_path, + func.count().label("cnt")) # type: ignore[call-overload] + .where(ThreatEvent.created_at >= cutoff).group_by( + ThreatEvent.request_path).order_by(func.count().desc()).limit(10)) path_rows = (await session.execute(path_q)).all() return StatsResponse( @@ -77,6 +74,10 @@ async def get_stats( medium=sev_map.get("MEDIUM", 0), low=sev_map.get("LOW", 0), ), - top_source_ips=[IPStatEntry(source_ip=row[0], count=row[1]) for row in ip_rows], - top_attacked_paths=[PathStatEntry(path=row[0], count=row[1]) for row in path_rows], + top_source_ips=[ + IPStatEntry(source_ip=row[0], count=row[1]) for row in ip_rows + ], + top_attacked_paths=[ + PathStatEntry(path=row[0], count=row[1]) for row in path_rows + ], ) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py b/PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py index 503d19ee..d7971dd1 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py @@ -9,6 +9,7 @@ from datetime import datetime from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession +from app.core.detection.ensemble import classify_severity from app.core.ingestion.pipeline import ScoredRequest from app.models.threat_event import ThreatEvent from app.schemas.threats import ( @@ -63,19 +64,28 @@ async def get_threats( count_query = select(func.count()).select_from(ThreatEvent) if severity: - query = query.where(ThreatEvent.severity == severity.upper()) # type: ignore[arg-type] - count_query = count_query.where(ThreatEvent.severity == severity.upper()) # type: ignore[arg-type] + query = query.where( + ThreatEvent.severity == severity.upper()) # type: ignore[arg-type] + count_query = count_query.where( + ThreatEvent.severity == severity.upper()) # type: ignore[arg-type] if source_ip: - query = query.where(ThreatEvent.source_ip == source_ip) # type: ignore[arg-type] - count_query = count_query.where(ThreatEvent.source_ip == source_ip) # type: ignore[arg-type] + query = query.where( + ThreatEvent.source_ip == source_ip) # type: ignore[arg-type] + count_query = count_query.where( + ThreatEvent.source_ip == source_ip) # type: ignore[arg-type] if since: - query = query.where(ThreatEvent.created_at >= since) # type: ignore[arg-type] - count_query = count_query.where(ThreatEvent.created_at >= since) # type: ignore[arg-type] + query = query.where(ThreatEvent.created_at + >= since) # type: ignore[arg-type] + count_query = count_query.where(ThreatEvent.created_at + >= since) # type: ignore[arg-type] if until: - query = query.where(ThreatEvent.created_at <= until) # type: ignore[arg-type] - count_query = count_query.where(ThreatEvent.created_at <= until) # type: ignore[arg-type] + query = query.where(ThreatEvent.created_at + <= until) # type: ignore[arg-type] + count_query = count_query.where(ThreatEvent.created_at + <= until) # type: ignore[arg-type] - query = query.order_by(ThreatEvent.created_at.desc()) # type: ignore[attr-defined] + query = query.order_by( + ThreatEvent.created_at.desc()) # type: ignore[attr-defined] query = query.offset(offset).limit(limit) total = (await session.execute(count_query)).scalar_one() @@ -116,16 +126,16 @@ async def create_threat_event( status_code=scored.entry.status_code, response_size=scored.entry.response_size, user_agent=scored.entry.user_agent, - threat_score=scored.rule_result.threat_score, - severity=scored.rule_result.severity, + threat_score=scored.final_score, + severity=classify_severity(scored.final_score), component_scores=scored.rule_result.component_scores, - geo_country=scored.geo.country if scored.geo else None, - geo_city=scored.geo.city if scored.geo else None, - geo_lat=scored.geo.lat if scored.geo else None, - geo_lon=scored.geo.lon if scored.geo else None, + geo_country=(scored.geo.country if scored.geo else None), + geo_city=(scored.geo.city if scored.geo else None), + geo_lat=(scored.geo.lat if scored.geo else None), + geo_lon=(scored.geo.lon if scored.geo else None), feature_vector=scored.feature_vector, - matched_rules=scored.rule_result.matched_rules or None, - model_version="rules-v1", + matched_rules=(scored.rule_result.matched_rules or None), + model_version=scored.detection_mode, ) session.add(event) await session.flush() diff --git a/PROJECTS/advanced/ai-threat-detection/backend/cli/main.py b/PROJECTS/advanced/ai-threat-detection/backend/cli/main.py index a3d44c00..22c074ef 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/cli/main.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/cli/main.py @@ -3,6 +3,8 @@ main.py """ +from pathlib import Path + import typer app = typer.Typer( @@ -11,15 +13,21 @@ app = typer.Typer( no_args_is_help=True, ) +DEFAULT_MODEL_DIR = "data/models" +DEFAULT_EPOCHS = 100 +DEFAULT_BATCH_SIZE = 256 +DEFAULT_SERVER_URL = "http://localhost:8000" + @app.command() def serve( host: str = typer.Option("0.0.0.0", help="Bind address"), port: int = typer.Option(8000, help="Bind port"), - reload: bool = typer.Option(False, help="Enable auto-reload for development"), + reload: bool = typer.Option(False, + help="Enable auto-reload for development"), ) -> None: """ - Start the AngelusVigil API server. + Start the AngelusVigil API server """ import uvicorn @@ -31,16 +39,148 @@ def serve( ) +@app.command() +def train( + dataset: Path = typer.Option(..., help="Path to CSIC 2010 CSV dataset"), + output_dir: Path = typer.Option( + DEFAULT_MODEL_DIR, + help="Directory to save ONNX models", + ), + epochs: int = typer.Option( + DEFAULT_EPOCHS, + help="Autoencoder training epochs", + ), + batch_size: int = typer.Option( + DEFAULT_BATCH_SIZE, + help="Training batch size", + ), +) -> None: + """ + Train all ML models and export to ONNX + """ + import json + + import numpy as np + from sklearn.model_selection import train_test_split + + from ml.export_onnx import ( + export_autoencoder, + export_isolation_forest, + export_random_forest, + ) + from ml.train_autoencoder import train_autoencoder + from ml.train_classifiers import ( + train_isolation_forest, + train_random_forest, + ) + + if not dataset.exists(): + typer.echo( + f"Error: dataset not found at {dataset}", + err=True, + ) + raise typer.Exit(code=1) + + output_dir.mkdir(parents=True, exist_ok=True) + + typer.echo(f"Loading dataset from {dataset}") + data = np.loadtxt(str(dataset), delimiter=",", dtype=np.float32) + X = data[:, :-1] + y = data[:, -1].astype(int) + + X_normal = X[y == 0] + X_train, X_test, y_train, y_test = train_test_split(X, + y, + test_size=0.2, + stratify=y, + random_state=42) + + typer.echo(f"Training autoencoder for {epochs} epochs") + ae_result = train_autoencoder( + X_normal, + epochs=epochs, + batch_size=batch_size, + ) + export_autoencoder(ae_result["model"], output_dir / "ae.onnx") + ae_result["scaler"].save_json(output_dir / "scaler.json") + threshold_data = {"threshold": float(ae_result["threshold"])} + (output_dir / "threshold.json").write_text(json.dumps(threshold_data)) + typer.echo(f" AE threshold: {ae_result['threshold']:.6f}") + + typer.echo("Training random forest") + rf_result = train_random_forest(X_train, y_train) + export_random_forest(rf_result["model"], X.shape[1], + output_dir / "rf.onnx") + typer.echo(f" RF F1: {rf_result['metrics']['f1']:.4f}") + + typer.echo("Training isolation forest") + if_result = train_isolation_forest(X_normal) + export_isolation_forest(if_result["model"], X.shape[1], + output_dir / "if.onnx") + typer.echo(f" IF samples: {if_result['metrics']['n_samples']}") + + typer.echo(f"Models exported to {output_dir}") + + +@app.command() +def replay( + log_file: Path = typer.Option(..., + help="Path to nginx access log file"), + url: str = typer.Option( + DEFAULT_SERVER_URL, + help="Running server URL to send logs to", + ), + batch_size: int = typer.Option(100, help="Lines per batch"), +) -> None: + """ + Replay historical log lines through the pipeline + """ + import httpx + + if not log_file.exists(): + typer.echo( + f"Error: log file not found at {log_file}", + err=True, + ) + raise typer.Exit(code=1) + + lines = log_file.read_text().strip().splitlines() + typer.echo(f"Replaying {len(lines)} lines to {url}") + + sent = 0 + with httpx.Client(timeout=30.0) as client: + for i in range(0, len(lines), batch_size): + batch = lines[i:i + batch_size] + response = client.post( + f"{url}/ingest/batch", + json={"lines": batch}, + ) + if response.status_code == 200: + sent += len(batch) + else: + typer.echo( + f" Batch {i} failed: {response.status_code}", + err=True, + ) + + typer.echo(f"Replayed {sent}/{len(lines)} lines") + + @app.command() def config() -> None: """ - Print the current configuration (secrets redacted). + Print the current configuration (secrets redacted) """ from app.config import settings safe_fields = {} for key, value in settings.model_dump().items(): - if any(secret in key for secret in ("key", "password", "secret", "token")): + if any(secret in key for secret in ( + "key", + "password", + "secret", + "token", + )): safe_fields[key] = "***REDACTED***" elif "url" in key and "@" in str(value): safe_fields[key] = _redact_url(str(value)) @@ -54,12 +194,12 @@ def config() -> None: @app.command() def health( url: str = typer.Option( - "http://localhost:8000", + DEFAULT_SERVER_URL, help="Base URL of the running server", ), ) -> None: """ - Ping the running server's /health endpoint. + Ping the running server's /health endpoint """ import httpx @@ -69,18 +209,24 @@ def health( data = response.json() typer.echo(f" status: {data.get('status', 'unknown')}") typer.echo(f" uptime: {data.get('uptime_seconds', 0):.0f}s") - typer.echo(f" pipeline: {'running' if data.get('pipeline_running') else 'stopped'}") + typer.echo( + f" pipeline: {'running' if data.get('pipeline_running') else 'stopped'}" + ) except httpx.ConnectError: typer.echo("Error: cannot connect to server", err=True) raise typer.Exit(code=1) from None except httpx.HTTPStatusError as exc: - typer.echo(f"Error: server returned {exc.response.status_code}", err=True) + typer.echo( + f"Error: server returned {exc.response.status_code}", + err=True, + ) raise typer.Exit(code=1) from None def _redact_url(url: str) -> str: """ - Replace the user:password portion of a database URL with ***:***. + Replace the user:password portion of a database + URL with ***:*** """ if "://" not in url or "@" not in url: return url diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/autoencoder.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/autoencoder.py index 9be7ba4e..af0fed64 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/ml/autoencoder.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/autoencoder.py @@ -71,4 +71,4 @@ class ThreatAutoencoder(nn.Module): Per-sample mean squared error between input and reconstruction. """ reconstructed = self.forward(x) - return torch.mean((x - reconstructed) ** 2, dim=1) + return torch.mean((x - reconstructed)**2, dim=1) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/experiment.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/experiment.py new file mode 100644 index 00000000..c1758ad2 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/experiment.py @@ -0,0 +1,104 @@ +""" +©AngelaMos | 2026 +experiment.py +""" + +import platform +import subprocess +import sys +from pathlib import Path +from types import TracebackType + +import mlflow + + +class VigilExperiment: + """ + Context manager wrapping MLflow experiment runs + with automatic system metadata logging + """ + + def __init__(self, experiment_name: str) -> None: + self._experiment_name = experiment_name + self._run: mlflow.ActiveRun | None = None + self._run_id: str | None = None + + @property + def run_id(self) -> str | None: + """ + The MLflow run ID, set after entering context + """ + return self._run_id + + def __enter__(self) -> VigilExperiment: + mlflow.set_experiment(self._experiment_name) + self._run = mlflow.start_run() + self._run_id = self._run.info.run_id + self._log_system_metadata() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + if exc_type is not None: + mlflow.set_tag("status", "failed") + mlflow.set_tag("error", str(exc_val)[:500]) + else: + mlflow.set_tag("status", "completed") + mlflow.end_run() + self._run = None + + def log_params(self, params: dict[str, object]) -> None: + """ + Log a dictionary of parameters to the active run + """ + mlflow.log_params(params) + + def log_metrics( + self, + metrics: dict[str, float], + step: int | None = None, + ) -> None: + """ + Log a dictionary of metrics to the active run + """ + mlflow.log_metrics(metrics, step=step) + + def log_artifact(self, path: Path | str) -> None: + """ + Log a local file as an artifact + """ + mlflow.log_artifact(str(path)) + + def _log_system_metadata(self) -> None: + """ + Record Python version and git commit hash + """ + mlflow.set_tag("python_version", sys.version.split()[0]) + mlflow.set_tag("platform", platform.system()) + + git_hash = _get_git_hash() + if git_hash is not None: + mlflow.set_tag("git_commit", git_hash) + + +def _get_git_hash() -> str | None: + """ + Return the short git commit hash or None + """ + try: + result = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + timeout=5, + check=False, + ) + if result.returncode == 0: + return result.stdout.strip() + except FileNotFoundError: + pass + return None diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/export_onnx.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/export_onnx.py new file mode 100644 index 00000000..0c49861e --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/export_onnx.py @@ -0,0 +1,90 @@ +""" +©AngelaMos | 2026 +export_onnx.py +""" + +from pathlib import Path + +import torch +from skl2onnx import convert_sklearn +from skl2onnx.common.data_types import FloatTensorType +from sklearn.base import BaseEstimator + +from ml.autoencoder import ThreatAutoencoder + +ONNX_OPSET = 17 +SKL_TARGET_OPSET = {"": 17, "ai.onnx.ml": 3} + + +def export_autoencoder( + model: ThreatAutoencoder, + path: Path | str, + opset: int = ONNX_OPSET, +) -> Path: + """ + Export a PyTorch autoencoder to ONNX with dynamic batch dimension + """ + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + + model.eval() + dummy = torch.randn(1, model.input_dim) + + batch_dim = torch.export.Dim("batch_size", min=1) + + torch.onnx.export( + model, + dummy, + str(path), + opset_version=opset, + export_params=True, + do_constant_folding=True, + input_names=["features"], + output_names=["reconstructed"], + dynamic_shapes={"x": { + 0: batch_dim + }}, + ) + return path + + +def export_random_forest( + model: BaseEstimator, + n_features: int, + path: Path | str, +) -> Path: + """ + Export a sklearn random forest (or calibrated wrapper) to ONNX + """ + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + + initial_type = [("features", FloatTensorType([None, n_features]))] + onnx_model = convert_sklearn( + model, + initial_types=initial_type, + target_opset=SKL_TARGET_OPSET, + ) + path.write_bytes(onnx_model.SerializeToString()) + return path + + +def export_isolation_forest( + model: BaseEstimator, + n_features: int, + path: Path | str, +) -> Path: + """ + Export a sklearn isolation forest to ONNX + """ + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + + initial_type = [("features", FloatTensorType([None, n_features]))] + onnx_model = convert_sklearn( + model, + initial_types=initial_type, + target_opset=SKL_TARGET_OPSET, + ) + path.write_bytes(onnx_model.SerializeToString()) + return path diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/train_autoencoder.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/train_autoencoder.py new file mode 100644 index 00000000..5ecc24c6 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/train_autoencoder.py @@ -0,0 +1,115 @@ +""" +©AngelaMos | 2026 +train_autoencoder.py +""" + +from typing import Any + +import numpy as np +import torch +from torch.utils.data import DataLoader, TensorDataset + +from ml.autoencoder import ThreatAutoencoder +from ml.scaler import FeatureScaler + + +def train_autoencoder( + X_normal: np.ndarray, + epochs: int = 100, + batch_size: int = 256, + lr: float = 1e-3, + percentile: float = 99.5, + val_split: float = 0.15, + patience: int = 10, +) -> dict[str, Any]: + """ + Train the autoencoder on normal-only traffic and calibrate the + anomaly detection threshold. + """ + input_dim = X_normal.shape[1] + + split_idx = int(len(X_normal) * (1 - val_split)) + X_train_raw = X_normal[:split_idx] + X_val_raw = X_normal[split_idx:] + + scaler = FeatureScaler() + X_train_scaled = scaler.fit_transform(X_train_raw) + X_val_scaled = scaler.transform(X_val_raw) + + train_tensor = torch.from_numpy(X_train_scaled) + val_tensor = torch.from_numpy(X_val_scaled) + + train_loader = DataLoader( + TensorDataset(train_tensor), + batch_size=batch_size, + shuffle=True, + drop_last=len(train_tensor) > batch_size, + ) + + model = ThreatAutoencoder(input_dim=input_dim) + optimizer = torch.optim.AdamW(model.parameters(), + lr=lr, + weight_decay=1e-5, + betas=(0.9, 0.999)) + scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, + mode="min", + factor=0.5, + patience=5, + min_lr=1e-6) + + history: dict[str, list[float]] = {"train_loss": [], "val_loss": []} + best_val_loss = float("inf") + best_state = None + epochs_without_improvement = 0 + + for _epoch in range(epochs): + model.train() + epoch_loss = 0.0 + n_batches = 0 + + for (batch, ) in train_loader: + reconstructed = model(batch) + loss = torch.nn.functional.mse_loss(reconstructed, batch) + optimizer.zero_grad() + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) + optimizer.step() + epoch_loss += loss.item() + n_batches += 1 + + avg_train_loss = epoch_loss / max(n_batches, 1) + history["train_loss"].append(avg_train_loss) + + model.eval() + with torch.no_grad(): + val_reconstructed = model(val_tensor) + val_loss = torch.nn.functional.mse_loss(val_reconstructed, + val_tensor).item() + history["val_loss"].append(val_loss) + + scheduler.step(val_loss) + + if val_loss < best_val_loss: + best_val_loss = val_loss + best_state = {k: v.clone() for k, v in model.state_dict().items()} + epochs_without_improvement = 0 + else: + epochs_without_improvement += 1 + + if epochs_without_improvement >= patience: + break + + if best_state is not None: + model.load_state_dict(best_state) + + model.eval() + with torch.no_grad(): + val_errors = model.compute_reconstruction_error(val_tensor) + threshold = float(np.percentile(val_errors.numpy(), percentile)) + + return { + "model": model, + "scaler": scaler, + "threshold": threshold, + "history": history, + } diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/train_classifiers.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/train_classifiers.py new file mode 100644 index 00000000..6d2b3fed --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/train_classifiers.py @@ -0,0 +1,79 @@ +""" +©AngelaMos | 2026 +train_classifiers.py +""" + +from typing import Any + +import numpy as np +from sklearn.calibration import CalibratedClassifierCV +from sklearn.ensemble import IsolationForest, RandomForestClassifier +from sklearn.metrics import ( + accuracy_score, + average_precision_score, + f1_score, + precision_score, + recall_score, +) +from sklearn.model_selection import train_test_split + + +def train_random_forest( + X: np.ndarray, + y: np.ndarray, + n_estimators: int = 200, + max_depth: int = 20, + calibration_split: float = 0.2, +) -> dict[str, Any]: + """ + Train a random forest with isotonic probability calibration + """ + X_train, X_eval, y_train, y_eval = train_test_split( + X, y, test_size=calibration_split, stratify=y, random_state=42) + + base_rf = RandomForestClassifier( + n_estimators=n_estimators, + max_depth=max_depth, + class_weight="balanced", + random_state=42, + n_jobs=-1, + ) + + calibrated = CalibratedClassifierCV(base_rf, method="isotonic", cv=3) + calibrated.fit(X_train, y_train) + + y_pred = calibrated.predict(X_eval) + y_proba = calibrated.predict_proba(X_eval)[:, 1] + + metrics = { + "accuracy": float(accuracy_score(y_eval, y_pred)), + "precision": float(precision_score(y_eval, y_pred, zero_division=0)), + "recall": float(recall_score(y_eval, y_pred, zero_division=0)), + "f1": float(f1_score(y_eval, y_pred, zero_division=0)), + "pr_auc": float(average_precision_score(y_eval, y_proba)), + } + + return {"model": calibrated, "metrics": metrics} + + +def train_isolation_forest( + X_normal: np.ndarray, + n_estimators: int = 200, +) -> dict[str, Any]: + """ + Train an isolation forest on normal-only traffic + """ + iso = IsolationForest( + n_estimators=n_estimators, + contamination="auto", + random_state=42, + n_jobs=-1, + ) + iso.fit(X_normal) + + return { + "model": iso, + "metrics": { + "n_samples": len(X_normal) + }, + } diff --git a/PROJECTS/advanced/ai-threat-detection/backend/pyproject.toml b/PROJECTS/advanced/ai-threat-detection/backend/pyproject.toml index e89ff1d5..1da61cce 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/pyproject.toml +++ b/PROJECTS/advanced/ai-threat-detection/backend/pyproject.toml @@ -50,6 +50,7 @@ ml = [ "numpy>=2.4.2", "pandas>=2.2.0,<3", "imbalanced-learn>=0.14.1", + "onnxscript>=0.6.2", ] [project.scripts] diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/conftest.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/conftest.py index 3a6bd74b..399062b9 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/conftest.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/conftest.py @@ -1,6 +1,8 @@ """ ©AngelaMos | 2026 conftest.py + +Shared pytest fixtures for in-memory SQLite database and HTTPX test client setup. """ from collections.abc import AsyncIterator @@ -84,5 +86,6 @@ async def db_client(db_engine) -> AsyncIterator[AsyncClient]: test_app.dependency_overrides[get_session] = override_get_session transport = ASGITransport(app=test_app) - async with AsyncClient(transport=transport, base_url="http://test") as client: + async with AsyncClient(transport=transport, + base_url="http://test") as client: yield client diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_api.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_api.py index 45b33567..f8e83d70 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_api.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_api.py @@ -1,6 +1,8 @@ """ ©AngelaMos | 2026 test_api.py + +Tests the FastAPI REST endpoints for threats, stats, health, readiness, and model management. """ import uuid @@ -19,7 +21,8 @@ async def test_health_returns_200() -> None: Health endpoint returns 200 with status and uptime. """ transport = ASGITransport(app=app) - async with AsyncClient(transport=transport, base_url="http://test") as client: + async with AsyncClient(transport=transport, + base_url="http://test") as client: response = await client.get("/health") assert response.status_code == 200 @@ -36,7 +39,8 @@ async def test_health_returns_pipeline_status() -> None: Health response includes pipeline_running boolean. """ transport = ASGITransport(app=app) - async with AsyncClient(transport=transport, base_url="http://test") as client: + async with AsyncClient(transport=transport, + base_url="http://test") as client: response = await client.get("/health") data = response.json() @@ -49,7 +53,8 @@ async def test_ready_returns_check_structure() -> None: Readiness endpoint returns structured component checks. """ transport = ASGITransport(app=app) - async with AsyncClient(transport=transport, base_url="http://test") as client: + async with AsyncClient(transport=transport, + base_url="http://test") as client: response = await client.get("/ready") assert response.status_code in (200, 503) @@ -178,15 +183,16 @@ async def test_stats_empty_window(db_client) -> None: @pytest.mark.asyncio async def test_model_status() -> None: """ - GET /models/status returns rules-only detection mode. + GET /models/status returns rules detection mode """ transport = ASGITransport(app=app) - async with AsyncClient(transport=transport, base_url="http://test") as client: + async with AsyncClient(transport=transport, + base_url="http://test") as client: response = await client.get("/models/status") assert response.status_code == 200 data = response.json() - assert data["detection_mode"] == "rules-only" + assert data["detection_mode"] == "rules" assert data["active_models"] == [] @@ -196,7 +202,8 @@ async def test_retrain_returns_202() -> None: POST /models/retrain returns 202 Accepted with a job ID. """ transport = ASGITransport(app=app) - async with AsyncClient(transport=transport, base_url="http://test") as client: + async with AsyncClient(transport=transport, + base_url="http://test") as client: response = await client.post("/models/retrain") assert response.status_code == 202 diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_autoencoder.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_autoencoder.py index 5e57c5ff..2e7758c1 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_autoencoder.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_autoencoder.py @@ -1,6 +1,8 @@ """ ©AngelaMos | 2026 test_autoencoder.py + +Tests the ThreatAutoencoder architecture: shapes, output range, reconstruction error, and training behavior. """ import pytest @@ -12,18 +14,27 @@ from ml.autoencoder import ThreatAutoencoder class TestAutoencoderArchitecture: def test_output_shape_matches_input(self) -> None: + """ + Forward pass on a batch of 16 produces output matching input shape. + """ model = ThreatAutoencoder(input_dim=35) x = torch.randn(16, 35) out = model(x) assert out.shape == (16, 35) def test_bottleneck_dim_is_six(self) -> None: + """ + Encoder bottleneck compresses 35 features to a 6-dimensional latent vector. + """ model = ThreatAutoencoder(input_dim=35) x = torch.randn(4, 35) encoded = model.encode(x) assert encoded.shape == (4, 6) def test_single_sample_forward(self) -> None: + """ + Single-sample forward pass completes without error in eval mode. + """ model = ThreatAutoencoder(input_dim=35) model.eval() x = torch.randn(1, 35) @@ -32,6 +43,9 @@ class TestAutoencoderArchitecture: assert out.shape == (1, 35) def test_output_values_in_zero_one_range(self) -> None: + """ + Decoder output is bounded to [0, 1] by the final activation. + """ model = ThreatAutoencoder(input_dim=35) model.eval() x = torch.randn(32, 35) @@ -41,14 +55,20 @@ class TestAutoencoderArchitecture: assert out.max() <= 1.0 def test_reconstruction_error_shape(self) -> None: + """ + compute_reconstruction_error returns one scalar per sample in the batch. + """ model = ThreatAutoencoder(input_dim=35) model.eval() x = torch.randn(8, 35) with torch.no_grad(): errors = model.compute_reconstruction_error(x) - assert errors.shape == (8,) + assert errors.shape == (8, ) def test_reconstruction_error_positive(self) -> None: + """ + Reconstruction error is non-negative for all samples. + """ model = ThreatAutoencoder(input_dim=35) model.eval() x = torch.randn(8, 35) @@ -56,7 +76,11 @@ class TestAutoencoderArchitecture: errors = model.compute_reconstruction_error(x) assert (errors >= 0.0).all() - def test_trained_model_reconstructs_normal_better_than_anomaly(self) -> None: + def test_trained_model_reconstructs_normal_better_than_anomaly( + self) -> None: + """ + After training on normal data, reconstruction error is lower for normals than anomalies. + """ torch.manual_seed(42) model = ThreatAutoencoder(input_dim=35) optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) @@ -74,13 +98,17 @@ class TestAutoencoderArchitecture: model.eval() with torch.no_grad(): - normal_errors = model.compute_reconstruction_error(normal_data[:50]) + normal_errors = model.compute_reconstruction_error( + normal_data[:50]) anomaly_data = torch.rand(50, 35) * 3.0 - 1.0 anomaly_errors = model.compute_reconstruction_error(anomaly_data) assert anomaly_errors.mean() > normal_errors.mean() def test_eval_mode_disables_dropout(self) -> None: + """ + Identical inputs produce identical outputs in eval mode (dropout is off). + """ model = ThreatAutoencoder(input_dim=35) model.eval() x = torch.randn(4, 35) @@ -91,6 +119,9 @@ class TestAutoencoderArchitecture: @pytest.mark.parametrize("batch_size", [1, 8, 32, 128]) def test_variable_batch_sizes(self, batch_size: int) -> None: + """ + Output shape matches input for batch sizes 1, 8, 32, and 128. + """ model = ThreatAutoencoder(input_dim=35) model.eval() x = torch.randn(batch_size, 35) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_cli.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_cli.py new file mode 100644 index 00000000..17ab4c76 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_cli.py @@ -0,0 +1,81 @@ +""" +©AngelaMos | 2026 +test_cli.py + +Tests the Typer CLI commands: help output, argument validation, and error handling for missing paths. +""" + +from typer.testing import CliRunner + +from cli.main import app + +runner = CliRunner() + + +class TestCLICommands: + + def test_train_help(self) -> None: + """ + train --help exits cleanly and mentions 'dataset' in its output. + """ + result = runner.invoke(app, ["train", "--help"]) + assert result.exit_code == 0 + assert "dataset" in result.output.lower() + + def test_replay_help(self) -> None: + """ + replay --help exits cleanly and mentions 'log' in its output. + """ + result = runner.invoke(app, ["replay", "--help"]) + assert result.exit_code == 0 + assert "log" in result.output.lower() + + def test_serve_help(self) -> None: + """ + serve --help exits cleanly and mentions 'host' in its output. + """ + result = runner.invoke(app, ["serve", "--help"]) + assert result.exit_code == 0 + assert "host" in result.output.lower() + + def test_config_help(self) -> None: + """ + config --help exits cleanly. + """ + result = runner.invoke(app, ["config", "--help"]) + assert result.exit_code == 0 + + def test_health_help(self) -> None: + """ + health --help exits cleanly. + """ + result = runner.invoke(app, ["health", "--help"]) + assert result.exit_code == 0 + + def test_train_missing_dataset_fails(self) -> None: + """ + train with a non-existent dataset path exits with a non-zero code. + """ + result = runner.invoke( + app, + [ + "train", + "--dataset", + "/nonexistent/data.csv", + ], + ) + assert result.exit_code != 0 + + def test_replay_missing_log_fails(self) -> None: + """ + replay with a non-existent log file exits with a non-zero code. + """ + result = runner.invoke( + app, + [ + "replay", + "--log-file", + "/nonexistent/access.log", + ], + ) + assert result.exit_code != 0 diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_config_ml.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_config_ml.py index c926faf2..17a2203e 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_config_ml.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_config_ml.py @@ -1,31 +1,45 @@ """ ©AngelaMos | 2026 test_config_ml.py + +Tests ML-related settings defaults: detection mode, ensemble weights, model paths, and MLflow URI. """ from app.config import settings def test_default_detection_mode_is_rules() -> None: + """ + Default detection_mode is 'rules' before any ML models are loaded. + """ assert settings.detection_mode == "rules" def test_default_ensemble_weights_sum_to_one() -> None: - total = ( - settings.ensemble_weight_ae - + settings.ensemble_weight_rf - + settings.ensemble_weight_if - ) + """ + AE + RF + IF ensemble weights sum to exactly 1.0. + """ + total = (settings.ensemble_weight_ae + settings.ensemble_weight_rf + + settings.ensemble_weight_if) assert abs(total - 1.0) < 1e-6 def test_default_model_dir() -> None: + """ + Default model artifact directory is data/models. + """ assert settings.model_dir == "data/models" def test_default_ae_threshold_percentile() -> None: + """ + Default autoencoder threshold percentile is 99.5. + """ assert settings.ae_threshold_percentile == 99.5 def test_default_mlflow_tracking_uri() -> None: + """ + Default MLflow tracking URI uses local file storage. + """ assert settings.mlflow_tracking_uri == "file:./mlruns" diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_detection.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_detection.py index 87c24cfb..6666d03c 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_detection.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_detection.py @@ -1,6 +1,8 @@ """ ©AngelaMos | 2026 test_detection.py + +Tests the rule engine's threat scoring, severity classification, and attack pattern matching. """ from datetime import datetime, UTC diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_ensemble.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_ensemble.py new file mode 100644 index 00000000..bce084ec --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_ensemble.py @@ -0,0 +1,156 @@ +""" +©AngelaMos | 2026 +test_ensemble.py + +Tests ensemble score normalization, weighted fusion, ML/rule blending, and severity classification. +""" + +from app.core.detection.ensemble import ( + blend_scores, + classify_severity, + fuse_scores, + normalize_ae_score, + normalize_if_score, +) + + +class TestScoreNormalization: + + def test_ae_score_below_threshold(self) -> None: + """ + AE error below threshold maps to a score below 0.5. + """ + result = normalize_ae_score(0.05, threshold=0.10) + assert abs(result - 0.25) < 1e-6 + + def test_ae_score_above_double_threshold_caps_at_one(self) -> None: + """ + AE error at 3x threshold is capped at 1.0. + """ + result = normalize_ae_score(0.30, threshold=0.10) + assert result == 1.0 + + def test_ae_score_zero_error(self) -> None: + """ + Zero reconstruction error maps to a score of 0.0. + """ + result = normalize_ae_score(0.0, threshold=0.10) + assert result == 0.0 + + def test_if_score_negative(self) -> None: + """ + Negative IF score (anomalous region) maps above 0.5. + """ + result = normalize_if_score(-0.5) + assert abs(result - 0.75) < 1e-6 + + def test_if_score_positive(self) -> None: + """ + Positive IF score (normal region) maps below 0.5. + """ + result = normalize_if_score(0.5) + assert abs(result - 0.25) < 1e-6 + + def test_if_score_zero(self) -> None: + """ + Zero IF score maps to exactly 0.5. + """ + result = normalize_if_score(0.0) + assert abs(result - 0.5) < 1e-6 + + +class TestEnsembleFusion: + + def test_weighted_average(self) -> None: + """ + Fused score is the weighted average of AE, RF, and IF scores. + """ + scores = {"ae": 0.8, "rf": 0.6, "if": 0.5} + weights = {"ae": 0.4, "rf": 0.4, "if": 0.2} + result = fuse_scores(scores, weights) + expected = 0.8 * 0.4 + 0.6 * 0.4 + 0.5 * 0.2 + assert abs(result - expected) < 1e-6 + + def test_all_zero_scores(self) -> None: + """ + All-zero model scores fuse to 0.0. + """ + scores = {"ae": 0.0, "rf": 0.0, "if": 0.0} + weights = {"ae": 0.4, "rf": 0.4, "if": 0.2} + assert fuse_scores(scores, weights) == 0.0 + + def test_all_max_scores(self) -> None: + """ + All-one model scores fuse to 1.0. + """ + scores = {"ae": 1.0, "rf": 1.0, "if": 1.0} + weights = {"ae": 0.4, "rf": 0.4, "if": 0.2} + assert abs(fuse_scores(scores, weights) - 1.0) < 1e-6 + + def test_partial_models(self) -> None: + """ + Fusion works correctly with only two models present. + """ + scores = {"ae": 0.9, "rf": 0.7} + weights = {"ae": 0.5, "rf": 0.5} + expected = 0.9 * 0.5 + 0.7 * 0.5 + assert abs(fuse_scores(scores, weights) - expected) < 1e-6 + + +class TestBlendScores: + + def test_blend_with_rule_score(self) -> None: + """ + ML and rule scores blend according to the specified ml_weight. + """ + result = blend_scores(ml_score=0.7, rule_score=0.9, ml_weight=0.7) + expected = 0.7 * 0.7 + 0.9 * 0.3 + assert abs(result - expected) < 1e-6 + + def test_blend_full_ml_weight(self) -> None: + """ + ml_weight of 1.0 returns the pure ML score. + """ + result = blend_scores(ml_score=0.8, rule_score=0.2, ml_weight=1.0) + assert abs(result - 0.8) < 1e-6 + + def test_blend_full_rule_weight(self) -> None: + """ + ml_weight of 0.0 returns the pure rule score. + """ + result = blend_scores(ml_score=0.8, rule_score=0.2, ml_weight=0.0) + assert abs(result - 0.2) < 1e-6 + + def test_blend_clamped_to_one(self) -> None: + """ + Blended score is clamped to 1.0 even when both inputs are 1.0. + """ + result = blend_scores(ml_score=1.0, rule_score=1.0, ml_weight=0.5) + assert result <= 1.0 + + +class TestClassifySeverity: + + def test_high(self) -> None: + """ + Scores >= 0.7 classify as HIGH. + """ + assert classify_severity(0.8) == "HIGH" + assert classify_severity(0.7) == "HIGH" + assert classify_severity(1.0) == "HIGH" + + def test_medium(self) -> None: + """ + Scores in [0.5, 0.7) classify as MEDIUM. + """ + assert classify_severity(0.55) == "MEDIUM" + assert classify_severity(0.5) == "MEDIUM" + assert classify_severity(0.69) == "MEDIUM" + + def test_low(self) -> None: + """ + Scores below 0.5 classify as LOW. + """ + assert classify_severity(0.3) == "LOW" + assert classify_severity(0.0) == "LOW" + assert classify_severity(0.49) == "LOW" diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_experiment.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_experiment.py new file mode 100644 index 00000000..0bffa51d --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_experiment.py @@ -0,0 +1,101 @@ +""" +©AngelaMos | 2026 +test_experiment.py + +Tests the VigilExperiment MLflow wrapper: run lifecycle, param/metric logging, and status tagging. +""" + +from pathlib import Path + +import mlflow +import pytest + +from ml.experiment import VigilExperiment + + +@pytest.fixture(autouse=True) +def _mlflow_tmp(tmp_path: Path) -> None: + """ + Point MLflow at a temp directory for isolation + """ + mlflow.set_tracking_uri(f"file:{tmp_path}/mlruns") + + +class TestVigilExperiment: + + def test_creates_run_with_id(self) -> None: + """ + Entering the context manager creates an MLflow run with a non-None run ID. + """ + with VigilExperiment("test-exp") as exp: + assert exp.run_id is not None + + def test_run_id_is_none_before_enter(self) -> None: + """ + run_id is None until the context manager is entered. + """ + exp = VigilExperiment("test-exp") + assert exp.run_id is None + + def test_log_params(self) -> None: + """ + log_params writes key-value pairs to the MLflow run as strings. + """ + with VigilExperiment("test-exp") as exp: + exp.log_params({"lr": 0.001, "epochs": 10}) + run = mlflow.get_run(exp.run_id) + assert run.data.params["lr"] == "0.001" + assert run.data.params["epochs"] == "10" + + def test_log_metrics(self) -> None: + """ + log_metrics stores numeric values on the MLflow run. + """ + with VigilExperiment("test-exp") as exp: + exp.log_metrics({"f1": 0.95, "loss": 0.02}) + run = mlflow.get_run(exp.run_id) + assert run.data.metrics["f1"] == 0.95 + + def test_log_artifact(self, tmp_path: Path) -> None: + """ + log_artifact uploads a file so it appears in the run's artifact list. + """ + artifact = tmp_path / "dummy.txt" + artifact.write_text("test content") + with VigilExperiment("test-exp") as exp: + exp.log_artifact(artifact) + run_id = exp.run_id + client = mlflow.MlflowClient() + artifacts = client.list_artifacts(run_id) + names = [a.path for a in artifacts] + assert "dummy.txt" in names + + def test_system_metadata_logged(self) -> None: + """ + python_version and platform tags are added automatically on run start. + """ + with VigilExperiment("test-exp") as exp: + run = mlflow.get_run(exp.run_id) + assert "python_version" in run.data.tags + assert "platform" in run.data.tags + + def test_completed_status_on_success(self) -> None: + """ + Run tagged with status='completed' when the context exits cleanly. + """ + with VigilExperiment("test-exp") as exp: + run_id = exp.run_id + run = mlflow.get_run(run_id) + assert run.data.tags["status"] == "completed" + + def test_failed_status_on_exception(self) -> None: + """ + Run tagged with status='failed' and error message when an exception is raised. + """ + run_id = None + with pytest.raises(ValueError), VigilExperiment("test-exp") as exp: + run_id = exp.run_id + raise ValueError("boom") + run = mlflow.get_run(run_id) + assert run.data.tags["status"] == "failed" + assert "boom" in run.data.tags["error"] diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_export_onnx.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_export_onnx.py new file mode 100644 index 00000000..09c50569 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_export_onnx.py @@ -0,0 +1,131 @@ +""" +©AngelaMos | 2026 +test_export_onnx.py + +Tests ONNX export for the autoencoder, random forest, and isolation forest models. +""" + +from pathlib import Path + +import numpy as np +import onnxruntime as ort +import torch +from sklearn.ensemble import IsolationForest, RandomForestClassifier + +from ml.autoencoder import ThreatAutoencoder +from ml.export_onnx import ( + export_autoencoder, + export_isolation_forest, + export_random_forest, +) + + +class TestAutoencoderExport: + + def test_creates_onnx_file(self, tmp_path: Path) -> None: + """ + Exporting creates a non-empty .onnx file at the given path. + """ + model = ThreatAutoencoder(input_dim=35) + path = export_autoencoder(model, tmp_path / "ae.onnx") + assert path.exists() + assert path.stat().st_size > 0 + + def test_onnx_output_matches_pytorch(self, tmp_path: Path) -> None: + """ + ONNX inference output matches PyTorch forward pass within 1e-5 tolerance. + """ + torch.manual_seed(42) + model = ThreatAutoencoder(input_dim=35) + model.eval() + onnx_path = export_autoencoder(model, tmp_path / "ae.onnx") + + rng = np.random.default_rng(42) + x = rng.standard_normal((8, 35)).astype(np.float32) + + with torch.no_grad(): + pt_out = model(torch.from_numpy(x)).numpy() + + session = ort.InferenceSession(str(onnx_path)) + ort_out = session.run(None, {"features": x})[0] + + np.testing.assert_allclose(pt_out, ort_out, atol=1e-5) + + def test_dynamic_batch_dimension(self, tmp_path: Path) -> None: + """ + Exported model accepts variable batch sizes (1, 16, 64). + """ + model = ThreatAutoencoder(input_dim=35) + model.eval() + onnx_path = export_autoencoder(model, tmp_path / "ae.onnx") + session = ort.InferenceSession(str(onnx_path)) + + for batch_size in (1, 16, 64): + rng = np.random.default_rng(batch_size) + x = rng.standard_normal((batch_size, 35)).astype(np.float32) + out = session.run(None, {"features": x})[0] + assert out.shape == (batch_size, 35) + + +class TestRandomForestExport: + + def test_creates_onnx_file(self, tmp_path: Path) -> None: + """ + Exporting a fitted RandomForest creates a non-empty .onnx file. + """ + rng = np.random.default_rng(42) + rf = RandomForestClassifier(n_estimators=10, random_state=42) + X = rng.standard_normal((100, 35)).astype(np.float32) + y = np.concatenate([np.zeros(70, dtype=int), np.ones(30, dtype=int)]) + rf.fit(X, y) + path = export_random_forest(rf, 35, tmp_path / "rf.onnx") + assert path.exists() + assert path.stat().st_size > 0 + + def test_onnx_produces_valid_output(self, tmp_path: Path) -> None: + """ + ONNX inference returns class predictions and probabilities for each sample. + """ + rng = np.random.default_rng(42) + rf = RandomForestClassifier(n_estimators=10, random_state=42) + X = rng.standard_normal((100, 35)).astype(np.float32) + y = np.concatenate([np.zeros(70, dtype=int), np.ones(30, dtype=int)]) + rf.fit(X, y) + onnx_path = export_random_forest(rf, 35, tmp_path / "rf.onnx") + session = ort.InferenceSession(str(onnx_path)) + + x_test = rng.standard_normal((5, 35)).astype(np.float32) + result = session.run(None, {"features": x_test}) + assert len(result) == 2 + assert len(result[0]) == 5 + + +class TestIsolationForestExport: + + def test_creates_onnx_file(self, tmp_path: Path) -> None: + """ + Exporting a fitted IsolationForest creates a non-empty .onnx file. + """ + rng = np.random.default_rng(42) + iso = IsolationForest(n_estimators=10, random_state=42) + iso.fit(rng.standard_normal((100, 35)).astype(np.float32)) + path = export_isolation_forest(iso, 35, tmp_path / "if.onnx") + assert path.exists() + assert path.stat().st_size > 0 + + def test_onnx_scores_match_decision_function(self, tmp_path: Path) -> None: + """ + ONNX anomaly scores match sklearn decision_function within 1e-4 tolerance. + """ + rng = np.random.default_rng(42) + iso = IsolationForest(n_estimators=10, random_state=42) + X = rng.standard_normal((100, 35)).astype(np.float32) + iso.fit(X) + onnx_path = export_isolation_forest(iso, 35, tmp_path / "if.onnx") + session = ort.InferenceSession(str(onnx_path)) + + x_test = rng.standard_normal((10, 35)).astype(np.float32) + sk_decision = iso.decision_function(x_test) + ort_scores = session.run(None, {"features": x_test})[1].flatten() + + np.testing.assert_allclose(sk_decision, ort_scores, atol=1e-4) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_features.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_features.py index dcd496f5..c4e654bd 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_features.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_features.py @@ -1,6 +1,8 @@ """ ©AngelaMos | 2026 test_features.py + +Tests per-request feature extraction, Redis sliding-window aggregation, and feature encoding. """ from datetime import datetime, UTC @@ -79,16 +81,20 @@ def test_path_depth() -> None: Path depth counts non-empty segments between slashes. """ assert extract_request_features(_make_entry(path="/"))["path_depth"] == 0 - assert extract_request_features(_make_entry(path="/api"))["path_depth"] == 1 - assert extract_request_features(_make_entry(path="/api/v1/users"))["path_depth"] == 3 + assert extract_request_features( + _make_entry(path="/api"))["path_depth"] == 1 + assert extract_request_features( + _make_entry(path="/api/v1/users"))["path_depth"] == 3 def test_path_entropy_high_vs_low() -> None: """ Random-character paths have higher entropy than simple paths. """ - low = extract_request_features(_make_entry(path="/index.html"))["path_entropy"] - high = extract_request_features(_make_entry(path="/x8Kp2mQz7wR4vL1n"))["path_entropy"] + low = extract_request_features( + _make_entry(path="/index.html"))["path_entropy"] + high = extract_request_features( + _make_entry(path="/x8Kp2mQz7wR4vL1n"))["path_entropy"] assert high > low @@ -96,7 +102,8 @@ def test_query_string_features() -> None: """ Query param count and length are extracted correctly. """ - features = extract_request_features(_make_entry(query_string="page=1&sort=name&limit=50")) + features = extract_request_features( + _make_entry(query_string="page=1&sort=name&limit=50")) assert features["query_param_count"] == 3 assert features["query_string_length"] == len("page=1&sort=name&limit=50") @@ -110,11 +117,11 @@ def test_url_encoding_detection() -> None: Percent-encoded sequences are detected in path and query. """ encoded = extract_request_features( - _make_entry(path="/search", query_string="q=%27OR+1%3D1") - ) + _make_entry(path="/search", query_string="q=%27OR+1%3D1")) assert encoded["has_encoded_chars"] is True - clean = extract_request_features(_make_entry(path="/index.html", query_string="")) + clean = extract_request_features( + _make_entry(path="/index.html", query_string="")) assert clean["has_encoded_chars"] is False @@ -133,9 +140,12 @@ def test_status_class() -> None: """ Status class groups status codes into Nxx buckets. """ - assert extract_request_features(_make_entry(status_code=200))["status_class"] == "2xx" - assert extract_request_features(_make_entry(status_code=404))["status_class"] == "4xx" - assert extract_request_features(_make_entry(status_code=503))["status_class"] == "5xx" + assert extract_request_features( + _make_entry(status_code=200))["status_class"] == "2xx" + assert extract_request_features( + _make_entry(status_code=404))["status_class"] == "4xx" + assert extract_request_features( + _make_entry(status_code=503))["status_class"] == "5xx" def test_temporal_features() -> None: @@ -161,14 +171,13 @@ def test_ua_bot_detection() -> None: """ bot = extract_request_features( _make_entry( - user_agent="Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" - ) - ) + user_agent= + "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" + )) assert bot["is_known_bot"] is True normal = extract_request_features( - _make_entry(user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64)") - ) + _make_entry(user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64)")) assert normal["is_known_bot"] is False @@ -176,7 +185,8 @@ def test_ua_scanner_detection() -> None: """ Known vulnerability scanner user agents are flagged. """ - nikto = extract_request_features(_make_entry(user_agent="Mozilla/5.00 (Nikto/2.1.6)")) + nikto = extract_request_features( + _make_entry(user_agent="Mozilla/5.00 (Nikto/2.1.6)")) assert nikto["is_known_scanner"] is True sqlmap = extract_request_features(_make_entry(user_agent="sqlmap/1.8")) @@ -187,15 +197,17 @@ def test_attack_pattern_detection() -> None: """ SQLi, XSS, and path traversal patterns in paths are detected. """ - sqli = extract_request_features(_make_entry(path="/users", query_string="id=1' OR 1=1--")) + sqli = extract_request_features( + _make_entry(path="/users", query_string="id=1' OR 1=1--")) assert sqli["has_attack_pattern"] is True xss = extract_request_features( - _make_entry(path="/comment", query_string="body=") - ) + _make_entry(path="/comment", + query_string="body=")) assert xss["has_attack_pattern"] is True - traversal = extract_request_features(_make_entry(path="/static/../../etc/passwd")) + traversal = extract_request_features( + _make_entry(path="/static/../../etc/passwd")) assert traversal["has_attack_pattern"] is True clean = extract_request_features(_make_entry(path="/api/v1/health")) @@ -206,11 +218,12 @@ def test_special_char_ratio() -> None: """ Paths with many non-alphanumeric characters have higher ratios. """ - clean = extract_request_features(_make_entry(path="/api/users"))["special_char_ratio"] + clean = extract_request_features( + _make_entry(path="/api/users"))["special_char_ratio"] - noisy = extract_request_features(_make_entry(path="/"))[ - "special_char_ratio" - ] + noisy = extract_request_features( + _make_entry( + path="/"))["special_char_ratio"] assert noisy > clean @@ -219,20 +232,25 @@ def test_private_ip_detection() -> None: """ RFC 1918 and loopback addresses are classified as private. """ - assert extract_request_features(_make_entry(ip="192.168.1.1"))["is_private_ip"] is True + assert extract_request_features( + _make_entry(ip="192.168.1.1"))["is_private_ip"] is True - assert extract_request_features(_make_entry(ip="127.0.0.1"))["is_private_ip"] is True + assert extract_request_features( + _make_entry(ip="127.0.0.1"))["is_private_ip"] is True - assert extract_request_features(_make_entry(ip="8.8.8.8"))["is_private_ip"] is False + assert extract_request_features( + _make_entry(ip="8.8.8.8"))["is_private_ip"] is False def test_file_extension() -> None: """ File extension is extracted from the path. """ - assert extract_request_features(_make_entry(path="/style.css"))["file_extension"] == ".css" + assert extract_request_features( + _make_entry(path="/style.css"))["file_extension"] == ".css" - assert extract_request_features(_make_entry(path="/api/users"))["file_extension"] == "" + assert extract_request_features( + _make_entry(path="/api/users"))["file_extension"] == "" def test_country_code_passthrough() -> None: diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_geoip.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_geoip.py index 00a0fa43..30a6d080 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_geoip.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_geoip.py @@ -1,6 +1,8 @@ """ ©AngelaMos | 2026 test_geoip.py + +Tests the GeoIP lookup service including private IP handling and missing database fallback. """ from unittest.mock import MagicMock, patch @@ -108,7 +110,9 @@ async def test_lookup_missing_city_name(mock_reader) -> None: """ Responses with None city name are handled gracefully. """ - mock_reader.city.return_value = _mock_city_response(city_name=None, lat=0.0, lon=0.0) + mock_reader.city.return_value = _mock_city_response(city_name=None, + lat=0.0, + lon=0.0) service = GeoIPService.__new__(GeoIPService) service._reader = mock_reader diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_inference.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_inference.py new file mode 100644 index 00000000..91baf558 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_inference.py @@ -0,0 +1,142 @@ +""" +©AngelaMos | 2026 +test_inference.py + +Tests the InferenceEngine: model loading, predict output shapes, score ranges, and missing-model handling. +""" + +import json +from pathlib import Path + +import numpy as np +import pytest + +from ml.autoencoder import ThreatAutoencoder +from ml.export_onnx import ( + export_autoencoder, + export_isolation_forest, + export_random_forest, +) +from ml.scaler import FeatureScaler +from sklearn.ensemble import IsolationForest, RandomForestClassifier + +from app.core.detection.inference import InferenceEngine + + +@pytest.fixture +def model_dir(tmp_path: Path) -> Path: + """ + Create a temp directory with all 3 ONNX models + scaler + threshold + """ + rng = np.random.default_rng(42) + X = rng.standard_normal((200, 35)).astype(np.float32) + y = np.concatenate([np.zeros(140, dtype=int), np.ones(60, dtype=int)]) + + ae = ThreatAutoencoder(input_dim=35) + export_autoencoder(ae, tmp_path / "ae.onnx") + + rf = RandomForestClassifier(n_estimators=10, random_state=42) + rf.fit(X, y) + export_random_forest(rf, 35, tmp_path / "rf.onnx") + + iso = IsolationForest(n_estimators=10, random_state=42) + iso.fit(X[:140]) + export_isolation_forest(iso, 35, tmp_path / "if.onnx") + + scaler = FeatureScaler() + scaler.fit(X[:140]) + scaler.save_json(tmp_path / "scaler.json") + + threshold_data = {"threshold": 0.05} + (tmp_path / "threshold.json").write_text(json.dumps(threshold_data)) + + return tmp_path + + +class TestInferenceEngine: + + def test_loads_all_models(self, model_dir: Path) -> None: + """ + Engine reports is_loaded=True when all three ONNX models are present. + """ + engine = InferenceEngine(model_dir=str(model_dir)) + assert engine.is_loaded + + def test_returns_none_when_no_models(self) -> None: + """ + Engine reports is_loaded=False when the model directory does not exist. + """ + engine = InferenceEngine(model_dir="/nonexistent/path") + assert not engine.is_loaded + + def test_predict_returns_none_when_not_loaded(self) -> None: + """ + predict returns None when the engine has no models loaded. + """ + engine = InferenceEngine(model_dir="/nonexistent/path") + result = engine.predict(np.zeros((1, 35), dtype=np.float32)) + assert result is None + + def test_predict_returns_scores(self, model_dir: Path) -> None: + """ + predict returns a dict with ae, rf, and if score arrays. + """ + engine = InferenceEngine(model_dir=str(model_dir)) + rng = np.random.default_rng(99) + x = rng.standard_normal((4, 35)).astype(np.float32) + result = engine.predict(x) + assert result is not None + assert "ae" in result + assert "rf" in result + assert "if" in result + + def test_predict_ae_scores_are_positive(self, model_dir: Path) -> None: + """ + AE reconstruction error scores are non-negative for all samples. + """ + engine = InferenceEngine(model_dir=str(model_dir)) + rng = np.random.default_rng(99) + x = rng.standard_normal((4, 35)).astype(np.float32) + result = engine.predict(x) + assert result is not None + assert all(s >= 0.0 for s in result["ae"]) + + def test_predict_rf_probabilities_in_range(self, model_dir: Path) -> None: + """ + RF malicious-class probabilities are within [0, 1]. + """ + engine = InferenceEngine(model_dir=str(model_dir)) + rng = np.random.default_rng(99) + x = rng.standard_normal((4, 35)).astype(np.float32) + result = engine.predict(x) + assert result is not None + assert all(0.0 <= p <= 1.0 for p in result["rf"]) + + def test_predict_single_sample(self, model_dir: Path) -> None: + """ + predict works on a single sample and returns one score per model. + """ + engine = InferenceEngine(model_dir=str(model_dir)) + rng = np.random.default_rng(99) + x = rng.standard_normal((1, 35)).astype(np.float32) + result = engine.predict(x) + assert result is not None + assert len(result["ae"]) == 1 + assert len(result["rf"]) == 1 + assert len(result["if"]) == 1 + + def test_threshold_loaded(self, model_dir: Path) -> None: + """ + Autoencoder threshold is read from threshold.json on initialization. + """ + engine = InferenceEngine(model_dir=str(model_dir)) + assert engine.threshold == 0.05 + + def test_partial_models_not_loaded(self, tmp_path: Path) -> None: + """ + Engine with only the AE model present reports is_loaded=False. + """ + ae = ThreatAutoencoder(input_dim=35) + export_autoencoder(ae, tmp_path / "ae.onnx") + engine = InferenceEngine(model_dir=str(tmp_path)) + assert not engine.is_loaded diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_integration.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_integration.py index 28093b50..1fc612ce 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_integration.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_integration.py @@ -1,6 +1,8 @@ """ ©AngelaMos | 2026 test_integration.py + +End-to-end tests covering the full path from log file write through tailer, pipeline, and database storage. """ import asyncio @@ -22,29 +24,22 @@ from app.core.ingestion.pipeline import Pipeline from app.core.ingestion.tailer import LogTailer from app.models.threat_event import ThreatEvent -NORMAL_LINE = ( - "192.168.1.100 - - [11/Feb/2026:10:00:00 +0000] " - '"GET /index.html HTTP/1.1" 200 4523 "-" ' - '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' -) +NORMAL_LINE = ("192.168.1.100 - - [11/Feb/2026:10:00:00 +0000] " + '"GET /index.html HTTP/1.1" 200 4523 "-" ' + '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"') -SQLI_LINE = ( - "198.51.100.10 - - [11/Feb/2026:10:00:01 +0000] " - '"GET /search?q=1%27+OR+1=1-- HTTP/1.1" 200 5678 "-" ' - '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' -) +SQLI_LINE = ("198.51.100.10 - - [11/Feb/2026:10:00:01 +0000] " + '"GET /search?q=1%27+OR+1=1-- HTTP/1.1" 200 5678 "-" ' + '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"') XSS_LINE = ( "198.51.100.11 - - [11/Feb/2026:10:00:02 +0000] " '"GET /comment?text= HTTP/1.1" 200 3210 "-" ' - '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' -) + '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"') -PATH_TRAVERSAL_LINE = ( - "198.51.100.12 - - [11/Feb/2026:10:00:03 +0000] " - '"GET /../../etc/passwd HTTP/1.1" 400 230 "-" ' - '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' -) +PATH_TRAVERSAL_LINE = ("198.51.100.12 - - [11/Feb/2026:10:00:03 +0000] " + '"GET /../../etc/passwd HTTP/1.1" 400 230 "-" ' + '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"') def _write_lines(log_path: str, *lines: str) -> None: @@ -127,7 +122,8 @@ async def _poll_threat_count( for _ in range(int(timeout / 0.1)): await asyncio.sleep(0.1) async with session_factory() as session: - result = await session.execute(select(func.count()).select_from(ThreatEvent)) + result = await session.execute( + select(func.count()).select_from(ThreatEvent)) count = result.scalar_one() if count >= expected: return count @@ -163,15 +159,15 @@ async def test_only_medium_plus_stored(integration_env) -> None: lines = [ f"192.168.1.{i + 1} - - [11/Feb/2026:10:00:0{i} +0000] " f'"GET /page/{i} HTTP/1.1" 200 1234 "-" ' - f'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' - for i in range(5) + f'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' for i in range(5) ] _write_lines(env["log_path"], *lines) await asyncio.sleep(2.0) async with env["session_factory"]() as session: - result = await session.execute(select(func.count()).select_from(ThreatEvent)) + result = await session.execute( + select(func.count()).select_from(ThreatEvent)) count = result.scalar_one() assert count == 0 diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_ml_integration.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_ml_integration.py new file mode 100644 index 00000000..3e4a2af8 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_ml_integration.py @@ -0,0 +1,179 @@ +""" +©AngelaMos | 2026 +test_ml_integration.py +""" + +import json +from pathlib import Path + +import fakeredis.aioredis +import numpy as np +import pytest +from sklearn.ensemble import ( + IsolationForest, + RandomForestClassifier, +) + +from app.core.detection.inference import InferenceEngine +from app.core.detection.rules import RuleEngine +from app.core.ingestion.pipeline import ( + Pipeline, + ScoredRequest, +) +from ml.autoencoder import ThreatAutoencoder +from ml.export_onnx import ( + export_autoencoder, + export_isolation_forest, + export_random_forest, +) +from ml.scaler import FeatureScaler + +VALID_LINE = ("93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] " + '"GET /api/v1/users HTTP/1.1" 200 1234 ' + '"https://example.com" ' + '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"') + +SQLI_LINE = ("93.184.216.34 - - [11/Feb/2026:14:30:01 +0000] " + '"GET /users?id=1%27OR%201=1-- HTTP/1.1" 200 512 ' + '"-" "Mozilla/5.0"') + + +@pytest.fixture +def trained_model_dir(tmp_path: Path) -> Path: + """ + Create a temp directory with trained ONNX models, + scaler, and threshold + """ + rng = np.random.default_rng(42) + X = rng.standard_normal((200, 35)).astype(np.float32) + y = np.concatenate([np.zeros(140, dtype=int), np.ones(60, dtype=int)]) + + ae = ThreatAutoencoder(input_dim=35) + export_autoencoder(ae, tmp_path / "ae.onnx") + + rf = RandomForestClassifier(n_estimators=10, random_state=42) + rf.fit(X, y) + export_random_forest(rf, 35, tmp_path / "rf.onnx") + + iso = IsolationForest(n_estimators=10, random_state=42) + iso.fit(X[:140]) + export_isolation_forest(iso, 35, tmp_path / "if.onnx") + + scaler = FeatureScaler() + scaler.fit(X[:140]) + scaler.save_json(tmp_path / "scaler.json") + + threshold_data = {"threshold": 0.05} + (tmp_path / "threshold.json").write_text(json.dumps(threshold_data)) + + return tmp_path + + +async def _make_pipeline_with_ml( + results: list[ScoredRequest], + model_dir: Path, +) -> Pipeline: + """ + Build a Pipeline with ML inference engine wired in + """ + redis = fakeredis.aioredis.FakeRedis(decode_responses=True) + engine = InferenceEngine(model_dir=str(model_dir)) + + async def collect(sr: ScoredRequest) -> None: + results.append(sr) + + pipeline = Pipeline( + redis_client=redis, + rule_engine=RuleEngine(), + on_result=collect, + inference_engine=engine, + ) + await pipeline.start() + return pipeline + + +async def _make_pipeline_rules_only( + results: list[ScoredRequest], ) -> Pipeline: + """ + Build a Pipeline without ML inference engine + """ + redis = fakeredis.aioredis.FakeRedis(decode_responses=True) + + async def collect(sr: ScoredRequest) -> None: + results.append(sr) + + pipeline = Pipeline( + redis_client=redis, + rule_engine=RuleEngine(), + on_result=collect, + ) + await pipeline.start() + return pipeline + + +class TestMLIntegration: + + @pytest.mark.asyncio + async def test_pipeline_with_ml_sets_hybrid_mode( + self, trained_model_dir: Path) -> None: + results: list[ScoredRequest] = [] + pipeline = await _make_pipeline_with_ml(results, trained_model_dir) + + await pipeline.raw_queue.put(VALID_LINE) + await pipeline.stop() + + assert len(results) == 1 + assert results[0].detection_mode == "hybrid" + + @pytest.mark.asyncio + async def test_pipeline_with_ml_produces_final_score( + self, trained_model_dir: Path) -> None: + results: list[ScoredRequest] = [] + pipeline = await _make_pipeline_with_ml(results, trained_model_dir) + + await pipeline.raw_queue.put(VALID_LINE) + await pipeline.stop() + + assert len(results) == 1 + assert results[0].final_score >= 0.0 + assert results[0].final_score <= 1.0 + + @pytest.mark.asyncio + async def test_rules_only_uses_rule_score(self, ) -> None: + results: list[ScoredRequest] = [] + pipeline = await _make_pipeline_rules_only(results) + + await pipeline.raw_queue.put(VALID_LINE) + await pipeline.stop() + + assert len(results) == 1 + assert results[0].detection_mode == "rules" + assert results[0].final_score == results[0].rule_result.threat_score + + @pytest.mark.asyncio + async def test_attack_scores_higher_than_benign( + self, trained_model_dir: Path) -> None: + results: list[ScoredRequest] = [] + pipeline = await _make_pipeline_with_ml(results, trained_model_dir) + + await pipeline.raw_queue.put(VALID_LINE) + await pipeline.raw_queue.put(SQLI_LINE) + await pipeline.stop() + + assert len(results) == 2 + benign_score = results[0].final_score + attack_score = results[1].final_score + assert attack_score > benign_score + + @pytest.mark.asyncio + async def test_rule_result_preserved_in_hybrid_mode( + self, trained_model_dir: Path) -> None: + results: list[ScoredRequest] = [] + pipeline = await _make_pipeline_with_ml(results, trained_model_dir) + + await pipeline.raw_queue.put(SQLI_LINE) + await pipeline.stop() + + assert len(results) == 1 + assert "SQL_INJECTION" in results[0].rule_result.matched_rules + assert results[0].rule_result.threat_score > 0 diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_parsers.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_parsers.py index fa5d031a..01a2639b 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_parsers.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_parsers.py @@ -1,6 +1,8 @@ """ ©AngelaMos | 2026 test_parsers.py + +Tests nginx combined log line parsing via parse_combined. """ from datetime import datetime, UTC @@ -12,12 +14,10 @@ def test_parse_standard_combined_line() -> None: """ Parse a standard nginx combined log line into all fields. """ - line = ( - "93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] " - '"GET /api/users?page=1 HTTP/1.1" 200 1234 ' - '"https://example.com" ' - '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' - ) + line = ("93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] " + '"GET /api/users?page=1 HTTP/1.1" 200 1234 ' + '"https://example.com" ' + '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"') result = parse_combined(line) assert result is not None @@ -52,10 +52,8 @@ def test_parse_missing_referer() -> None: """ A dash referer is normalized to an empty string. """ - line = ( - "10.0.0.1 - - [11/Feb/2026:08:15:42 +0000] " - '"POST /login HTTP/1.1" 302 0 "-" "Mozilla/5.0"' - ) + line = ("10.0.0.1 - - [11/Feb/2026:08:15:42 +0000] " + '"POST /login HTTP/1.1" 302 0 "-" "Mozilla/5.0"') result = parse_combined(line) assert result is not None @@ -71,8 +69,7 @@ def test_parse_complex_query_string() -> None: line = ( "93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] " '"GET /search?q=hello+world&lang=en&page=2&sort=relevance HTTP/1.1" ' - '200 5678 "https://example.com/search" "Mozilla/5.0"' - ) + '200 5678 "https://example.com/search" "Mozilla/5.0"') result = parse_combined(line) assert result is not None @@ -110,11 +107,9 @@ def test_parse_full_ipv6_address() -> None: """ Parse a line with a full-length IPv6 address. """ - line = ( - "2001:0db8:85a3:0000:0000:8a2e:0370:7334 - - " - '[11/Feb/2026:14:30:00 +0000] "GET /api/v1/health HTTP/2.0" ' - '200 256 "-" "python-httpx/0.28"' - ) + line = ("2001:0db8:85a3:0000:0000:8a2e:0370:7334 - - " + '[11/Feb/2026:14:30:00 +0000] "GET /api/v1/health HTTP/2.0" ' + '200 256 "-" "python-httpx/0.28"') result = parse_combined(line) assert result is not None diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_pipeline.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_pipeline.py index e294251e..5befebb3 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_pipeline.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_pipeline.py @@ -1,6 +1,8 @@ """ ©AngelaMos | 2026 test_pipeline.py + +Tests the async ingestion pipeline: parsing, feature extraction, rule scoring, and shutdown. """ import fakeredis.aioredis @@ -9,18 +11,14 @@ import pytest from app.core.detection.rules import RuleEngine from app.core.ingestion.pipeline import Pipeline, ScoredRequest -VALID_LINE = ( - "93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] " - '"GET /api/v1/users HTTP/1.1" 200 1234 ' - '"https://example.com" ' - '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' -) +VALID_LINE = ("93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] " + '"GET /api/v1/users HTTP/1.1" 200 1234 ' + '"https://example.com" ' + '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"') -SQLI_LINE = ( - "93.184.216.34 - - [11/Feb/2026:14:30:01 +0000] " - '"GET /users?id=1%27OR%201=1-- HTTP/1.1" 200 512 ' - '"-" "Mozilla/5.0"' -) +SQLI_LINE = ("93.184.216.34 - - [11/Feb/2026:14:30:01 +0000] " + '"GET /users?id=1%27OR%201=1-- HTTP/1.1" 200 512 ' + '"-" "Mozilla/5.0"') async def _make_pipeline( diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_scaler.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_scaler.py index 1fb05906..2125455a 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_scaler.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_scaler.py @@ -1,6 +1,8 @@ """ ©AngelaMos | 2026 test_scaler.py + +Tests the FeatureScaler: fitting, transform correctness, JSON serialization, and round-trip loading. """ import json @@ -14,6 +16,9 @@ from ml.scaler import FeatureScaler @pytest.fixture def sample_data() -> np.ndarray: + """ + Two-hundred samples of 35-dimensional random float32 data. + """ rng = np.random.default_rng(42) return rng.standard_normal((200, 35)).astype(np.float32) @@ -21,23 +26,36 @@ def sample_data() -> np.ndarray: class TestFeatureScaler: def test_fit_sets_n_features(self, sample_data: np.ndarray) -> None: + """ + Fitting stores the number of input features. + """ scaler = FeatureScaler() scaler.fit(sample_data) assert scaler.n_features == 35 def test_transform_preserves_shape(self, sample_data: np.ndarray) -> None: + """ + Transform output has the same shape as the input array. + """ scaler = FeatureScaler() scaler.fit(sample_data) transformed = scaler.transform(sample_data) assert transformed.shape == sample_data.shape def test_transform_dtype_float32(self, sample_data: np.ndarray) -> None: + """ + Transformed array dtype remains float32. + """ scaler = FeatureScaler() scaler.fit(sample_data) transformed = scaler.transform(sample_data) assert transformed.dtype == np.float32 - def test_transformed_median_near_zero(self, sample_data: np.ndarray) -> None: + def test_transformed_median_near_zero(self, + sample_data: np.ndarray) -> None: + """ + Median of each feature column is approximately zero after scaling. + """ scaler = FeatureScaler() scaler.fit(sample_data) transformed = scaler.transform(sample_data) @@ -45,17 +63,21 @@ class TestFeatureScaler: assert np.allclose(medians, 0.0, atol=0.15) def test_inverse_transform_recovers_original( - self, sample_data: np.ndarray - ) -> None: + self, sample_data: np.ndarray) -> None: + """ + Inverse transform recovers the original values within floating-point tolerance. + """ scaler = FeatureScaler() scaler.fit(sample_data) transformed = scaler.transform(sample_data) recovered = scaler.inverse_transform(transformed) np.testing.assert_allclose(recovered, sample_data, atol=1e-5) - def test_save_json_creates_file( - self, sample_data: np.ndarray, tmp_path: Path - ) -> None: + def test_save_json_creates_file(self, sample_data: np.ndarray, + tmp_path: Path) -> None: + """ + save_json writes a non-empty file to the given path. + """ scaler = FeatureScaler() scaler.fit(sample_data) path = tmp_path / "scaler.json" @@ -63,9 +85,11 @@ class TestFeatureScaler: assert path.exists() assert path.stat().st_size > 0 - def test_save_json_is_valid_json( - self, sample_data: np.ndarray, tmp_path: Path - ) -> None: + def test_save_json_is_valid_json(self, sample_data: np.ndarray, + tmp_path: Path) -> None: + """ + Saved JSON contains center, scale, and n_features keys. + """ scaler = FeatureScaler() scaler.fit(sample_data) path = tmp_path / "scaler.json" @@ -75,9 +99,11 @@ class TestFeatureScaler: assert "scale" in data assert "n_features" in data - def test_load_json_round_trip( - self, sample_data: np.ndarray, tmp_path: Path - ) -> None: + def test_load_json_round_trip(self, sample_data: np.ndarray, + tmp_path: Path) -> None: + """ + Loading from JSON produces a scaler with identical transform output. + """ scaler = FeatureScaler() scaler.fit(sample_data) path = tmp_path / "scaler.json" @@ -91,11 +117,17 @@ class TestFeatureScaler: np.testing.assert_allclose(original_out, loaded_out, atol=1e-6) def test_transform_before_fit_raises(self) -> None: + """ + Calling transform before fit raises RuntimeError. + """ scaler = FeatureScaler() with pytest.raises(RuntimeError): scaler.transform(np.zeros((5, 35), dtype=np.float32)) def test_fit_transform_convenience(self, sample_data: np.ndarray) -> None: + """ + fit_transform fits and transforms in one call. + """ scaler = FeatureScaler() result = scaler.fit_transform(sample_data) assert result.shape == sample_data.shape diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_training.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_training.py index 69d4eaa9..a5d32b14 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_training.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_training.py @@ -1,6 +1,8 @@ """ ©AngelaMos | 2026 test_training.py + +Tests training pipelines for the autoencoder, random forest, and isolation forest models. """ import numpy as np @@ -14,10 +16,18 @@ class TestAutoencoderTraining: @pytest.fixture def normal_data(self) -> np.ndarray: + """ + Three-hundred samples of clipped 35-dimensional float32 data representing normal traffic. + """ rng = np.random.default_rng(42) - return (rng.standard_normal((300, 35)) * 0.3 + 0.5).astype(np.float32).clip(0, 1) + return (rng.standard_normal( + (300, 35)) * 0.3 + 0.5).astype(np.float32).clip(0, 1) - def test_returns_model_and_threshold(self, normal_data: np.ndarray) -> None: + def test_returns_model_and_threshold(self, + normal_data: np.ndarray) -> None: + """ + Training returns model, threshold, scaler, and history keys. + """ result = train_autoencoder(normal_data, epochs=5, batch_size=32) assert "model" in result assert "threshold" in result @@ -25,24 +35,38 @@ class TestAutoencoderTraining: assert "history" in result def test_threshold_is_positive(self, normal_data: np.ndarray) -> None: + """ + Computed reconstruction error threshold is a positive value. + """ result = train_autoencoder(normal_data, epochs=5, batch_size=32) assert result["threshold"] > 0.0 def test_history_has_train_loss(self, normal_data: np.ndarray) -> None: + """ + Training history includes one train_loss entry per epoch. + """ result = train_autoencoder(normal_data, epochs=5, batch_size=32) assert "train_loss" in result["history"] assert len(result["history"]["train_loss"]) == 5 def test_custom_percentile(self, normal_data: np.ndarray) -> None: - result_95 = train_autoencoder( - normal_data, epochs=3, batch_size=32, percentile=95.0 - ) - result_99 = train_autoencoder( - normal_data, epochs=3, batch_size=32, percentile=99.0 - ) + """ + Higher percentile produces a higher or equal reconstruction threshold. + """ + result_95 = train_autoencoder(normal_data, + epochs=3, + batch_size=32, + percentile=95.0) + result_99 = train_autoencoder(normal_data, + epochs=3, + batch_size=32, + percentile=99.0) assert result_99["threshold"] >= result_95["threshold"] def test_model_is_in_eval_mode(self, normal_data: np.ndarray) -> None: + """ + Returned model is in eval mode after training completes. + """ result = train_autoencoder(normal_data, epochs=3, batch_size=32) assert not result["model"].training @@ -51,37 +75,50 @@ class TestRandomForestTraining: @pytest.fixture def labeled_data(self) -> tuple[np.ndarray, np.ndarray]: + """ + Four-hundred samples with 300 benign and 100 attack labels. + """ rng = np.random.default_rng(42) X = rng.standard_normal((400, 35)).astype(np.float32) - y = np.concatenate([np.zeros(300, dtype=np.int64), np.ones(100, dtype=np.int64)]) + y = np.concatenate( + [np.zeros(300, dtype=np.int64), + np.ones(100, dtype=np.int64)]) return X, y def test_returns_model_and_metrics( - self, labeled_data: tuple[np.ndarray, np.ndarray] - ) -> None: + self, labeled_data: tuple[np.ndarray, np.ndarray]) -> None: + """ + Training returns model and metrics dict. + """ X, y = labeled_data result = train_random_forest(X, y) assert "model" in result assert "metrics" in result def test_model_has_predict_proba( - self, labeled_data: tuple[np.ndarray, np.ndarray] - ) -> None: + self, labeled_data: tuple[np.ndarray, np.ndarray]) -> None: + """ + Trained model exposes predict_proba for probability scoring. + """ X, y = labeled_data result = train_random_forest(X, y) assert hasattr(result["model"], "predict_proba") def test_metrics_contain_required_keys( - self, labeled_data: tuple[np.ndarray, np.ndarray] - ) -> None: + self, labeled_data: tuple[np.ndarray, np.ndarray]) -> None: + """ + Metrics dict contains f1, pr_auc, accuracy, precision, and recall. + """ X, y = labeled_data result = train_random_forest(X, y) for key in ("f1", "pr_auc", "accuracy", "precision", "recall"): assert key in result["metrics"] def test_probabilities_in_valid_range( - self, labeled_data: tuple[np.ndarray, np.ndarray] - ) -> None: + self, labeled_data: tuple[np.ndarray, np.ndarray]) -> None: + """ + Predicted probabilities are within the [0, 1] range. + """ X, y = labeled_data result = train_random_forest(X, y) proba = result["model"].predict_proba(X[:10]) @@ -89,8 +126,10 @@ class TestRandomForestTraining: assert proba.max() <= 1.0 def test_metrics_values_in_valid_range( - self, labeled_data: tuple[np.ndarray, np.ndarray] - ) -> None: + self, labeled_data: tuple[np.ndarray, np.ndarray]) -> None: + """ + All metric values fall within [0, 1]. + """ X, y = labeled_data result = train_random_forest(X, y) for value in result["metrics"].values(): @@ -101,24 +140,39 @@ class TestIsolationForestTraining: @pytest.fixture def normal_data(self) -> np.ndarray: + """ + Two-hundred samples of 35-dimensional standard normal float32 data. + """ rng = np.random.default_rng(42) return rng.standard_normal((200, 35)).astype(np.float32) def test_returns_model(self, normal_data: np.ndarray) -> None: + """ + Training returns a model key in the result dict. + """ result = train_isolation_forest(normal_data) assert "model" in result def test_model_has_score_samples(self, normal_data: np.ndarray) -> None: + """ + Trained model exposes score_samples for anomaly scoring. + """ result = train_isolation_forest(normal_data) assert hasattr(result["model"], "score_samples") - def test_returns_metrics_with_n_samples(self, normal_data: np.ndarray) -> None: + def test_returns_metrics_with_n_samples(self, + normal_data: np.ndarray) -> None: + """ + Metrics include n_samples matching the training set size. + """ result = train_isolation_forest(normal_data) assert result["metrics"]["n_samples"] == 200 def test_anomaly_scores_distinguish_normal_and_outlier( - self, normal_data: np.ndarray - ) -> None: + self, normal_data: np.ndarray) -> None: + """ + Normal training data scores higher than extreme outliers. + """ result = train_isolation_forest(normal_data) model = result["model"] normal_scores = model.score_samples(normal_data[:50]) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/uv.lock b/PROJECTS/advanced/ai-threat-detection/backend/uv.lock index 055a3d1e..6a3b6df9 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/uv.lock +++ b/PROJECTS/advanced/ai-threat-detection/backend/uv.lock @@ -136,6 +136,7 @@ ml = [ { name = "mlflow" }, { name = "numpy" }, { name = "onnxruntime" }, + { name = "onnxscript" }, { name = "pandas" }, { name = "scikit-learn" }, { name = "skl2onnx" }, @@ -157,6 +158,7 @@ requires-dist = [ { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.19.1" }, { name = "numpy", marker = "extra == 'ml'", specifier = ">=2.4.2" }, { name = "onnxruntime", marker = "extra == 'ml'", specifier = ">=1.24.1" }, + { name = "onnxscript", marker = "extra == 'ml'", specifier = ">=0.6.2" }, { name = "orjson", specifier = ">=3.11.7" }, { name = "pandas", marker = "extra == 'ml'", specifier = ">=2.2.0,<3" }, { name = "pydantic", specifier = ">=2.12.5" }, @@ -1589,6 +1591,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/01/a0/4fb0e6d36eaf079af366b2c1f68bafe92df6db963e2295da84388af64abc/onnx-1.20.1-cp312-abi3-win_arm64.whl", hash = "sha256:21d747348b1c8207406fa2f3e12b82f53e0d5bb3958bcd0288bd27d3cb6ebb00", size = 16344155, upload-time = "2026-01-10T01:39:45.536Z" }, ] +[[package]] +name = "onnx-ir" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy" }, + { name = "onnx" }, + { name = "sympy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b2/a5/acc43c8fa6edbc584d127fb6bbd13ae9ebfc01b9675c74e0da2de15fa4a6/onnx_ir-0.2.0.tar.gz", hash = "sha256:8bad3906691987290789b26d05e0dbff467029a0b1e411e12e4cae02e43503e4", size = 141693, upload-time = "2026-02-24T02:31:10.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/df/a99736bcca6b16e36c687ce4996abcf4ce73c514fddd9e730cfcb6a334f2/onnx_ir-0.2.0-py3-none-any.whl", hash = "sha256:eb14d1399c2442bd1ff702719e70074e9cedfa3af5729416a32752c9e0f82591", size = 164100, upload-time = "2026-02-24T02:31:09.454Z" }, +] + [[package]] name = "onnxruntime" version = "1.24.1" @@ -1609,6 +1627,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/a2/cfcf009eb38d90cc628c087b6506b3dfe1263387f3cbbf8d272af4fef957/onnxruntime-1.24.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34488aa760fb5c2e6d06a7ca9241124eb914a6a06f70936a14c669d1b3df9598", size = 17099815, upload-time = "2026-02-05T17:31:43.092Z" }, ] +[[package]] +name = "onnxscript" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy" }, + { name = "onnx" }, + { name = "onnx-ir" }, + { name = "packaging" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/2b/538fdeb0e25bed5d7e0f954af5710543e2629499fb74381afc3333f8a8ae/onnxscript-0.6.2.tar.gz", hash = "sha256:abb2e6f464db40c9b8c7fbb3e64cca04cf3f4495e67c4eda5eac17b784191ce3", size = 590865, upload-time = "2026-02-10T22:53:39.638Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/56/e6b179397497ab93266b6eb00743403a6a699a29063a423c4a14595d3db9/onnxscript-0.6.2-py3-none-any.whl", hash = "sha256:20e3c3fd1da19b3655549d5455a2df719db47374fe430e01e865ae69127c37b9", size = 689064, upload-time = "2026-02-10T22:53:41.663Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.39.1" diff --git a/PROJECTS/advanced/ai-threat-detection/justfile b/PROJECTS/advanced/ai-threat-detection/justfile index 19643a5f..00438df8 100644 --- a/PROJECTS/advanced/ai-threat-detection/justfile +++ b/PROJECTS/advanced/ai-threat-detection/justfile @@ -24,20 +24,19 @@ default: [group('lint')] ruff *ARGS: - cd backend && uv run ruff check app/ cli/ {{ARGS}} + cd backend && uv run ruff check app/ cli/ ml/ tests/ {{ARGS}} [group('lint')] ruff-fix: - cd backend && uv run ruff check app/ cli/ --fix - cd backend && uv run ruff format app/ cli/ + cd backend && uv run ruff check app/ cli/ ml/ tests/ --fix [group('lint')] -ruff-format: - cd backend && uv run ruff format app/ cli/ +yapf *ARGS: + cd backend && uv run yapf -rip app/ cli/ ml/ tests/ {{ARGS}} [group('lint')] pylint *ARGS: - cd backend && uv run pylint app {{ARGS}} + cd backend && uv run pylint app cli ml {{ARGS}} [group('lint')] lint: ruff pylint @@ -48,7 +47,7 @@ lint: ruff pylint [group('types')] mypy *ARGS: - cd backend && uv run mypy app/ cli/ {{ARGS}} + cd backend && uv run mypy app/ cli/ ml/ {{ARGS}} [group('types')] typecheck: mypy @@ -64,9 +63,13 @@ pytest *ARGS: [group('test')] test: pytest +[group('test')] +test-v *ARGS: + cd backend && uv run pytest tests/ -v {{ARGS}} + [group('test')] test-cov: - cd backend && uv run pytest tests/ --cov=app --cov-report=term-missing --cov-report=html + cd backend && uv run pytest tests/ --cov=app --cov=cli --cov=ml --cov-report=term-missing --cov-report=html # ============================================================================= # CI / Quality @@ -222,6 +225,18 @@ vigil-config: vigil-health: curl -sf http://localhost:8000/health | python -m json.tool +[group('vigil')] +vigil-train *ARGS: + cd backend && uv run vigil train {{ARGS}} + +[group('vigil')] +vigil-replay *ARGS: + cd backend && uv run vigil replay {{ARGS}} + +[group('vigil')] +vigil-status: + curl -sf http://localhost:8000/models/status | python -m json.tool + # ============================================================================= # Setup # ============================================================================= @@ -233,8 +248,8 @@ setup: setup-backend env [group('setup')] setup-backend: @echo "Setting up backend..." - cd backend && uv sync - @echo "Backend setup complete!" + cd backend && uv sync --all-groups + @echo "Backend setup complete (with dev + ml deps)!" [group('setup')] env: From 2475d1603f9dde055b3c4dc0fb171a71623c84f7 Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Sat, 28 Feb 2026 05:39:51 -0500 Subject: [PATCH 02/19] feat(ml): add CSIC 2010 data loader with feature extraction Parses multi-line CSIC 2010 HTTP request files into CSICRequest dataclasses, converts to ParsedLogEntry with synthesized defaults, and produces 35-dim feature vectors via the existing extraction pipeline. --- .../backend/ml/data_loader.py | 220 +++++++++++ .../backend/tests/test_data_loader.py | 354 ++++++++++++++++++ 2 files changed, 574 insertions(+) create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/ml/data_loader.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/tests/test_data_loader.py diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/data_loader.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/data_loader.py new file mode 100644 index 00000000..c283cd29 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/data_loader.py @@ -0,0 +1,220 @@ +""" +©AngelaMos | 2026 +data_loader.py +""" + +import logging +import re +from dataclasses import dataclass +from datetime import datetime, UTC +from pathlib import Path + +import numpy as np + +from app.core.features.encoder import encode_for_inference +from app.core.features.extractor import extract_request_features +from app.core.ingestion.parsers import ParsedLogEntry + +logger = logging.getLogger(__name__) + +_REQUEST_LINE_RE = re.compile( + r"^(GET|POST|PUT|DELETE|HEAD|OPTIONS|PATCH|TRACE)" + r"\s+(\S+)\s+(HTTP/\d\.\d)\s*$" +) + +_DEFAULT_IP = "192.168.1.100" + +_DEFAULT_UA = ( + "Mozilla/5.0 (compatible; Konqueror/3.5; Linux)" + " KHTML/3.5.8 (like Gecko)" +) + +_WINDOWED_FEATURE_NAMES: list[str] = [ + "req_count_1m", + "req_count_5m", + "req_count_10m", + "error_rate_5m", + "unique_paths_5m", + "unique_uas_10m", + "method_entropy_5m", + "avg_response_size_5m", + "status_diversity_5m", + "path_depth_variance_5m", + "inter_request_time_mean", + "inter_request_time_std", +] + + +@dataclass +class CSICRequest: + """ + Single HTTP request parsed from CSIC 2010 dataset format + """ + + method: str + path: str + query_string: str + protocol: str + headers: dict[str, str] + body: str + label: int + + +def parse_csic_file( + path: Path, + label: int, +) -> list[CSICRequest]: + """ + Parse a CSIC 2010 dataset file into a list of CSICRequest objects + """ + text = path.read_text(encoding="utf-8", errors="replace") + lines = text.splitlines() + + blocks: list[list[str]] = [] + current: list[str] = [] + + for line in lines: + match = _REQUEST_LINE_RE.match(line) + if match and current: + blocks.append(current) + current = [line] + elif match: + current = [line] + elif current: + current.append(line) + + if current: + blocks.append(current) + + results: list[CSICRequest] = [] + for block in blocks: + req = _parse_request_block(block, label) + if req is not None: + results.append(req) + + logger.info( + "Parsed %d requests from %s (label=%d)", + len(results), + path.name, + label, + ) + return results + + +def _parse_request_block( + lines: list[str], + label: int, +) -> CSICRequest | None: + """ + Parse a single request block into a CSICRequest + """ + if not lines: + return None + + match = _REQUEST_LINE_RE.match(lines[0]) + if not match: + return None + + method = match.group(1) + full_uri = match.group(2) + protocol = match.group(3) + + if "?" in full_uri: + path, query_string = full_uri.split("?", 1) + else: + path = full_uri + query_string = "" + + headers: dict[str, str] = {} + body_start = len(lines) + + for i, line in enumerate(lines[1:], 1): + if not line.strip(): + body_start = i + 1 + break + if ": " in line: + key, value = line.split(": ", 1) + headers[key] = value + + body_lines = [ + ln for ln in lines[body_start:] if ln.strip() + ] + body = "\n".join(body_lines) + + return CSICRequest( + method=method, + path=path, + query_string=query_string, + protocol=protocol, + headers=headers, + body=body, + label=label, + ) + + +def csic_to_parsed_entry(req: CSICRequest) -> ParsedLogEntry: + """ + Convert a CSICRequest to a ParsedLogEntry with synthesized defaults + for fields not present in the CSIC dataset + """ + ua = req.headers.get("User-Agent", _DEFAULT_UA) + + query = req.query_string + if req.body: + query = ( + f"{query}&{req.body}" if query else req.body + ) + + return ParsedLogEntry( + ip=_DEFAULT_IP, + timestamp=datetime.now(UTC), + method=req.method, + path=req.path, + query_string=query, + status_code=200, + response_size=0, + referer="", + user_agent=ua, + raw_line="", + ) + + +def load_csic_dataset( + normal_path: Path, + attack_path: Path, +) -> tuple[np.ndarray, np.ndarray]: + """ + Load CSIC 2010 normal and attack files, extract features, + and return (X, y) arrays ready for model training + """ + normal_reqs = parse_csic_file(normal_path, label=0) + attack_reqs = parse_csic_file(attack_path, label=1) + + all_reqs = normal_reqs + attack_reqs + + vectors: list[list[float]] = [] + labels: list[int] = [] + + for req in all_reqs: + entry = csic_to_parsed_entry(req) + features = extract_request_features(entry) + + for name in _WINDOWED_FEATURE_NAMES: + features[name] = 0.0 + + vector = encode_for_inference(features) + vectors.append(vector) + labels.append(req.label) + + X = np.array(vectors, dtype=np.float32) + y = np.array(labels, dtype=np.int32) + + logger.info( + "Dataset loaded: X=%s, y=%s (normal=%d, attack=%d)", + X.shape, + y.shape, + np.sum(y == 0), + np.sum(y == 1), + ) + + return X, y diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_data_loader.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_data_loader.py new file mode 100644 index 00000000..b15073d7 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_data_loader.py @@ -0,0 +1,354 @@ +""" +©AngelaMos | 2026 +test_data_loader.py +""" + +from pathlib import Path + +import numpy as np + +from ml.data_loader import ( + CSICRequest, + csic_to_parsed_entry, + load_csic_dataset, + parse_csic_file, +) + +NORMAL_GET_FIXTURE = """\ +GET /tienda1/publico/anadir.jsp?id=2&nombre=Jam%F3n HTTP/1.1 +Host: localhost:8080 +User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko) +Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5 +Accept-Language: en +Accept-Charset: iso-8859-1,*,utf-8 +Accept-Encoding: x-gzip, x-deflate, gzip, deflate +Connection: close + +GET /tienda1/publico/pagar.jsp HTTP/1.1 +Host: localhost:8080 +User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko) +Accept: text/xml,application/xml +Accept-Language: en +Connection: close + +GET /tienda1/publico/entrar.jsp HTTP/1.1 +Host: localhost:8080 +User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko) +Accept: text/xml +Connection: close + +POST /tienda1/publico/registro.jsp HTTP/1.1 +Host: localhost:8080 +User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko) +Accept: text/xml,application/xml +Content-Type: application/x-www-form-urlencoded +Content-Length: 64 +Connection: close + +nombre=Juan&apellidos=Garcia&email=juan@example.com&submit=Enviar + +GET /tienda1/publico/vac498.jsp?manufacturer=Dell HTTP/1.1 +Host: localhost:8080 +User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko) +Accept: text/xml +Connection: close +""" + +ATTACK_FIXTURE = """\ +GET /tienda1/publico/anadir.jsp?id=2&nombre=Jam%F3n'+OR+1=1-- HTTP/1.1 +Host: localhost:8080 +User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko) +Accept: text/xml +Connection: close + +POST /tienda1/publico/autenticar.jsp HTTP/1.1 +Host: localhost:8080 +User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko) +Content-Type: application/x-www-form-urlencoded +Content-Length: 68 +Connection: close + +usuario=admin'+OR+1=1--&contrasenya=pass&B1=Enviar + +GET /tienda1/publico/../../../etc/passwd HTTP/1.1 +Host: localhost:8080 +User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko) +Accept: text/xml +Connection: close + +GET /tienda1/publico/anadir.jsp?id= HTTP/1.1 +Host: localhost:8080 +User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko) +Accept: text/xml +Connection: close +""" + +MALFORMED_FIXTURE = """\ +THIS IS NOT HTTP + +GET /valid/path HTTP/1.1 +Host: localhost:8080 +User-Agent: TestBot +Connection: close + +just some random garbage here +and more garbage +""" + + +class TestParseCSICFile: + """ + Test CSIC 2010 file parser + """ + + def test_parses_normal_get_requests(self, tmp_path: Path) -> None: + """ + Normal GET requests are parsed into CSICRequest dataclass instances + """ + f = tmp_path / "normalTrafficTraining.txt" + f.write_text(NORMAL_GET_FIXTURE, encoding="utf-8") + + results = parse_csic_file(f, label=0) + + assert len(results) == 5 + assert all(isinstance(r, CSICRequest) for r in results) + assert all(r.label == 0 for r in results) + + def test_parses_get_method_and_path(self, tmp_path: Path) -> None: + """ + First GET request has correct method, path, and query string + """ + f = tmp_path / "normal.txt" + f.write_text(NORMAL_GET_FIXTURE, encoding="utf-8") + + results = parse_csic_file(f, label=0) + first = results[0] + + assert first.method == "GET" + assert first.path == "/tienda1/publico/anadir.jsp" + assert first.query_string == "id=2&nombre=Jam%F3n" + assert first.protocol == "HTTP/1.1" + + def test_parses_headers(self, tmp_path: Path) -> None: + """ + Headers are captured as a dict + """ + f = tmp_path / "normal.txt" + f.write_text(NORMAL_GET_FIXTURE, encoding="utf-8") + + results = parse_csic_file(f, label=0) + first = results[0] + + assert first.headers["Host"] == "localhost:8080" + assert "Konqueror" in first.headers["User-Agent"] + + def test_parses_post_with_body(self, tmp_path: Path) -> None: + """ + POST request body is captured + """ + f = tmp_path / "normal.txt" + f.write_text(NORMAL_GET_FIXTURE, encoding="utf-8") + + results = parse_csic_file(f, label=0) + post_req = results[3] + + assert post_req.method == "POST" + assert "nombre=Juan" in post_req.body + + def test_parses_attack_file_with_label_1( + self, tmp_path: Path) -> None: + """ + Attack file entries get label=1 + """ + f = tmp_path / "anomalous.txt" + f.write_text(ATTACK_FIXTURE, encoding="utf-8") + + results = parse_csic_file(f, label=1) + + assert len(results) == 4 + assert all(r.label == 1 for r in results) + + def test_attack_sqli_in_query(self, tmp_path: Path) -> None: + """ + SQLi payload appears in query string of attack GET + """ + f = tmp_path / "anomalous.txt" + f.write_text(ATTACK_FIXTURE, encoding="utf-8") + + results = parse_csic_file(f, label=1) + first = results[0] + + assert "OR+1=1" in first.query_string + + def test_attack_sqli_in_body(self, tmp_path: Path) -> None: + """ + SQLi payload appears in POST body of attack + """ + f = tmp_path / "anomalous.txt" + f.write_text(ATTACK_FIXTURE, encoding="utf-8") + + results = parse_csic_file(f, label=1) + post_req = results[1] + + assert post_req.method == "POST" + assert "OR+1=1" in post_req.body + + def test_malformed_blocks_skipped( + self, tmp_path: Path) -> None: + """ + Malformed/non-HTTP lines are skipped gracefully + """ + f = tmp_path / "malformed.txt" + f.write_text(MALFORMED_FIXTURE, encoding="utf-8") + + results = parse_csic_file(f, label=0) + + assert len(results) == 1 + assert results[0].method == "GET" + assert results[0].path == "/valid/path" + + def test_empty_file_returns_empty_list( + self, tmp_path: Path) -> None: + """ + Empty file returns an empty list + """ + f = tmp_path / "empty.txt" + f.write_text("", encoding="utf-8") + + results = parse_csic_file(f, label=0) + + assert results == [] + + +class TestCSICToParsedEntry: + """ + Test conversion from CSICRequest to ParsedLogEntry + """ + + def test_converts_get_request(self) -> None: + """ + GET CSICRequest converts to ParsedLogEntry with synthesized fields + """ + req = CSICRequest( + method="GET", + path="/tienda1/publico/anadir.jsp", + query_string="id=2&nombre=test", + protocol="HTTP/1.1", + headers={ + "Host": "localhost:8080", + "User-Agent": "Mozilla/5.0 (compatible; Konqueror/3.5)", + }, + body="", + label=0, + ) + + entry = csic_to_parsed_entry(req) + + assert entry.method == "GET" + assert entry.path == "/tienda1/publico/anadir.jsp" + assert entry.query_string == "id=2&nombre=test" + assert entry.status_code == 200 + assert entry.response_size == 0 + assert entry.user_agent == "Mozilla/5.0 (compatible; Konqueror/3.5)" + assert entry.ip != "" + assert entry.timestamp is not None + + def test_converts_post_with_body_in_query(self) -> None: + """ + POST body is appended to query_string for feature extraction + """ + req = CSICRequest( + method="POST", + path="/login", + query_string="", + protocol="HTTP/1.1", + headers={"User-Agent": "TestBot"}, + body="user=admin'+OR+1=1--&pass=x", + label=1, + ) + + entry = csic_to_parsed_entry(req) + + assert entry.method == "POST" + assert "OR+1=1" in entry.query_string + + def test_missing_ua_gets_default(self) -> None: + """ + CSICRequest without User-Agent header gets a default user agent + """ + req = CSICRequest( + method="GET", + path="/test", + query_string="", + protocol="HTTP/1.1", + headers={"Host": "localhost"}, + body="", + label=0, + ) + + entry = csic_to_parsed_entry(req) + + assert len(entry.user_agent) > 0 + + +class TestLoadCSICDataset: + """ + Test end-to-end dataset loading with feature extraction + """ + + def test_returns_correct_shape(self, tmp_path: Path) -> None: + """ + load_csic_dataset returns X with 35 columns and matching y + """ + normal = tmp_path / "normal.txt" + attack = tmp_path / "attack.txt" + normal.write_text(NORMAL_GET_FIXTURE, encoding="utf-8") + attack.write_text(ATTACK_FIXTURE, encoding="utf-8") + + X, y = load_csic_dataset(normal, attack) + + assert X.shape[1] == 35 + assert X.shape[0] == y.shape[0] + + def test_contains_both_labels(self, tmp_path: Path) -> None: + """ + y array contains both 0 (normal) and 1 (attack) labels + """ + normal = tmp_path / "normal.txt" + attack = tmp_path / "attack.txt" + normal.write_text(NORMAL_GET_FIXTURE, encoding="utf-8") + attack.write_text(ATTACK_FIXTURE, encoding="utf-8") + + _, y = load_csic_dataset(normal, attack) + + assert 0 in y + assert 1 in y + + def test_label_counts_match_files( + self, tmp_path: Path) -> None: + """ + Normal and attack counts match the number of requests in each file + """ + normal = tmp_path / "normal.txt" + attack = tmp_path / "attack.txt" + normal.write_text(NORMAL_GET_FIXTURE, encoding="utf-8") + attack.write_text(ATTACK_FIXTURE, encoding="utf-8") + + _, y = load_csic_dataset(normal, attack) + + assert np.sum(y == 0) == 5 + assert np.sum(y == 1) == 4 + + def test_feature_values_are_finite( + self, tmp_path: Path) -> None: + """ + All feature values are finite (no NaN or Inf) + """ + normal = tmp_path / "normal.txt" + attack = tmp_path / "attack.txt" + normal.write_text(NORMAL_GET_FIXTURE, encoding="utf-8") + attack.write_text(ATTACK_FIXTURE, encoding="utf-8") + + X, _ = load_csic_dataset(normal, attack) + + assert np.all(np.isfinite(X)) From 0a9cf12212937c1a4f3ba4c5db42bac538d00e30 Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Sat, 28 Feb 2026 05:59:36 -0500 Subject: [PATCH 03/19] feat(ml): add ensemble validation with PR-AUC and F1 gates Loads ONNX models, normalizes per-model scores, fuses them, and computes classification metrics with configurable quality gates. --- .../backend/ml/validation.py | 154 ++++++++++++++ .../backend/tests/test_validation.py | 189 ++++++++++++++++++ 2 files changed, 343 insertions(+) create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/ml/validation.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/tests/test_validation.py diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/validation.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/validation.py new file mode 100644 index 00000000..d1755005 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/validation.py @@ -0,0 +1,154 @@ +""" +©AngelaMos | 2026 +validation.py +""" + +import logging +from dataclasses import dataclass, field +from pathlib import Path + +import numpy as np +from sklearn.metrics import ( + average_precision_score, + confusion_matrix, + f1_score, + precision_score, + recall_score, + roc_auc_score, +) + +from app.core.detection.ensemble import ( + fuse_scores, + normalize_ae_score, + normalize_if_score, +) +from app.core.detection.inference import InferenceEngine + +logger = logging.getLogger(__name__) + +DEFAULT_ENSEMBLE_WEIGHTS: dict[str, float] = { + "ae": 0.4, + "rf": 0.4, + "if": 0.2, +} + +BINARY_THRESHOLD = 0.5 + + +@dataclass +class ValidationResult: + """ + Ensemble validation metrics and gate results + """ + + precision: float + recall: float + f1: float + pr_auc: float + roc_auc: float + confusion_matrix: list[list[int]] + passed_gates: bool + gate_details: dict[str, bool] = field(default_factory=dict) + + +def validate_ensemble( + model_dir: Path, + X_test: np.ndarray, + y_test: np.ndarray, + ensemble_weights: dict[str, float] | None = None, + pr_auc_gate: float = 0.85, + f1_gate: float = 0.80, +) -> ValidationResult: + """ + Run all 3 models on test data and compute classification metrics + """ + weights = ensemble_weights or DEFAULT_ENSEMBLE_WEIGHTS + + engine = InferenceEngine(model_dir=str(model_dir)) + if not engine.is_loaded: + raise RuntimeError( + f"Failed to load models from {model_dir}" + ) + + raw_scores = engine.predict( + X_test.astype(np.float32), + ) + if raw_scores is None: + raise RuntimeError("Inference returned None") + + fused = _compute_fused_scores( + raw_scores, engine.threshold, weights + ) + + y_pred = (fused >= BINARY_THRESHOLD).astype(np.int32) + + prec = float( + precision_score(y_test, y_pred, zero_division=0) + ) + rec = float( + recall_score(y_test, y_pred, zero_division=0) + ) + f1_val = float( + f1_score(y_test, y_pred, zero_division=0) + ) + pr_auc_val = float( + average_precision_score(y_test, fused) + ) + roc_auc_val = float(roc_auc_score(y_test, fused)) + + cm = confusion_matrix(y_test, y_pred).tolist() + + pr_auc_passed = pr_auc_val >= pr_auc_gate + f1_passed = f1_val >= f1_gate + gate_details = { + "pr_auc": pr_auc_passed, + "f1": f1_passed, + } + + logger.info( + "Validation: precision=%.3f recall=%.3f " + "f1=%.3f pr_auc=%.3f roc_auc=%.3f", + prec, + rec, + f1_val, + pr_auc_val, + roc_auc_val, + ) + + return ValidationResult( + precision=prec, + recall=rec, + f1=f1_val, + pr_auc=pr_auc_val, + roc_auc=roc_auc_val, + confusion_matrix=cm, + passed_gates=pr_auc_passed and f1_passed, + gate_details=gate_details, + ) + + +def _compute_fused_scores( + raw_scores: dict[str, list[float]], + threshold: float, + weights: dict[str, float], +) -> np.ndarray: + """ + Normalize and fuse per-model raw scores into a single score array + """ + n_samples = len(raw_scores["ae"]) + fused = np.zeros(n_samples, dtype=np.float64) + + for i in range(n_samples): + per_model: dict[str, float] = {} + + per_model["ae"] = normalize_ae_score( + raw_scores["ae"][i], threshold + ) + per_model["rf"] = raw_scores["rf"][i] + per_model["if"] = normalize_if_score( + raw_scores["if"][i] + ) + + fused[i] = fuse_scores(per_model, weights) + + return fused diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_validation.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_validation.py new file mode 100644 index 00000000..14926e43 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_validation.py @@ -0,0 +1,189 @@ +""" +©AngelaMos | 2026 +test_validation.py +""" + +import json +from pathlib import Path + +import numpy as np +import pytest +from sklearn.ensemble import IsolationForest, RandomForestClassifier + +from ml.autoencoder import ThreatAutoencoder +from ml.export_onnx import ( + export_autoencoder, + export_isolation_forest, + export_random_forest, +) +from ml.scaler import FeatureScaler +from ml.validation import ValidationResult, validate_ensemble + + +@pytest.fixture +def trained_model_dir(tmp_path: Path) -> Path: + """ + Create a temp directory with trained ONNX models for validation testing + """ + rng = np.random.default_rng(42) + X_normal = rng.standard_normal((200, 35)).astype(np.float32) + X_attack = rng.standard_normal((80, 35)).astype(np.float32) + 3.0 + X = np.vstack([X_normal, X_attack]) + y = np.array([0] * 200 + [1] * 80, dtype=np.int32) + + ae = ThreatAutoencoder(input_dim=35) + export_autoencoder(ae, tmp_path / "ae.onnx") + + rf = RandomForestClassifier(n_estimators=10, random_state=42) + rf.fit(X, y) + export_random_forest(rf, 35, tmp_path / "rf.onnx") + + iso = IsolationForest(n_estimators=10, random_state=42) + iso.fit(X_normal) + export_isolation_forest(iso, 35, tmp_path / "if.onnx") + + scaler = FeatureScaler() + scaler.fit(X_normal) + scaler.save_json(tmp_path / "scaler.json") + + (tmp_path / "threshold.json").write_text( + json.dumps({"threshold": 0.05}) + ) + + return tmp_path + + +@pytest.fixture +def separable_test_data() -> tuple[np.ndarray, np.ndarray]: + """ + Test data where normal and attack clusters are well-separated + """ + rng = np.random.default_rng(99) + X_normal = rng.standard_normal((50, 35)).astype(np.float32) + X_attack = ( + rng.standard_normal((30, 35)).astype(np.float32) + 3.0 + ) + X = np.vstack([X_normal, X_attack]) + y = np.array([0] * 50 + [1] * 30, dtype=np.int32) + return X, y + + +class TestValidateEnsemble: + """ + Test ensemble validation with metric gates + """ + + def test_returns_validation_result( + self, + trained_model_dir: Path, + separable_test_data: tuple[np.ndarray, np.ndarray], + ) -> None: + """ + validate_ensemble returns a ValidationResult instance + """ + X_test, y_test = separable_test_data + result = validate_ensemble(trained_model_dir, X_test, y_test) + + assert isinstance(result, ValidationResult) + + def test_result_has_all_metrics( + self, + trained_model_dir: Path, + separable_test_data: tuple[np.ndarray, np.ndarray], + ) -> None: + """ + ValidationResult contains precision, recall, f1, pr_auc, roc_auc + """ + X_test, y_test = separable_test_data + result = validate_ensemble(trained_model_dir, X_test, y_test) + + assert 0.0 <= result.precision <= 1.0 + assert 0.0 <= result.recall <= 1.0 + assert 0.0 <= result.f1 <= 1.0 + assert 0.0 <= result.pr_auc <= 1.0 + assert 0.0 <= result.roc_auc <= 1.0 + + def test_confusion_matrix_shape( + self, + trained_model_dir: Path, + separable_test_data: tuple[np.ndarray, np.ndarray], + ) -> None: + """ + Confusion matrix is 2x2 + """ + X_test, y_test = separable_test_data + result = validate_ensemble(trained_model_dir, X_test, y_test) + + assert len(result.confusion_matrix) == 2 + assert len(result.confusion_matrix[0]) == 2 + assert len(result.confusion_matrix[1]) == 2 + + def test_gate_details_contains_both_gates( + self, + trained_model_dir: Path, + separable_test_data: tuple[np.ndarray, np.ndarray], + ) -> None: + """ + gate_details has pr_auc and f1 entries + """ + X_test, y_test = separable_test_data + result = validate_ensemble(trained_model_dir, X_test, y_test) + + assert "pr_auc" in result.gate_details + assert "f1" in result.gate_details + + def test_gates_pass_with_low_thresholds( + self, + trained_model_dir: Path, + separable_test_data: tuple[np.ndarray, np.ndarray], + ) -> None: + """ + passed_gates is True when gates are set very low + """ + X_test, y_test = separable_test_data + result = validate_ensemble( + trained_model_dir, + X_test, + y_test, + pr_auc_gate=0.01, + f1_gate=0.01, + ) + + assert result.passed_gates is True + + def test_gates_fail_with_high_thresholds( + self, + trained_model_dir: Path, + separable_test_data: tuple[np.ndarray, np.ndarray], + ) -> None: + """ + passed_gates is False when gates are impossibly high + """ + X_test, y_test = separable_test_data + result = validate_ensemble( + trained_model_dir, + X_test, + y_test, + pr_auc_gate=1.0, + f1_gate=1.0, + ) + + assert result.passed_gates is False + + def test_custom_ensemble_weights( + self, + trained_model_dir: Path, + separable_test_data: tuple[np.ndarray, np.ndarray], + ) -> None: + """ + Custom ensemble weights are accepted without error + """ + X_test, y_test = separable_test_data + result = validate_ensemble( + trained_model_dir, + X_test, + y_test, + ensemble_weights={"ae": 0.5, "rf": 0.3, "if": 0.2}, + ) + + assert isinstance(result, ValidationResult) From ee25636437e21f84f346cce1589c04087f05ac83 Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Sat, 28 Feb 2026 06:00:28 -0500 Subject: [PATCH 04/19] feat(ml): add model metadata persistence with version hashing SHA-256 based version hashing for ONNX artifacts and async metadata persistence that replaces previous active model entries. --- .../backend/ml/metadata.py | 82 +++++++ .../backend/tests/test_metadata.py | 212 ++++++++++++++++++ 2 files changed, 294 insertions(+) create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/ml/metadata.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/tests/test_metadata.py diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/metadata.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/metadata.py new file mode 100644 index 00000000..f7a23bed --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/metadata.py @@ -0,0 +1,82 @@ +""" +©AngelaMos | 2026 +metadata.py +""" + +import hashlib +import logging +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.model_metadata import ModelMetadata + +logger = logging.getLogger(__name__) + +MODEL_TYPES: dict[str, str] = { + "ae.onnx": "autoencoder", + "rf.onnx": "random_forest", + "if.onnx": "isolation_forest", +} + +VERSION_HASH_LENGTH = 12 + + +def compute_model_version(artifact_path: Path) -> str: + """ + Compute a 12-char hex version string from the SHA-256 hash of a file + """ + sha = hashlib.sha256(artifact_path.read_bytes()) + return sha.hexdigest()[:VERSION_HASH_LENGTH] + + +async def save_model_metadata( + session: AsyncSession, + model_dir: Path, + training_samples: int, + metrics: dict[str, object], + mlflow_run_id: str | None = None, + threshold: float | None = None, +) -> list[ModelMetadata]: + """ + Persist metadata for all 3 model types, deactivating previous active versions + """ + rows: list[ModelMetadata] = [] + + for filename, model_type in MODEL_TYPES.items(): + artifact_path = model_dir / filename + version = compute_model_version(artifact_path) + + result = await session.execute( + select(ModelMetadata).where( + ModelMetadata.model_type == model_type, + ModelMetadata.is_active == True, # noqa: E712 + ) + ) + for old in result.scalars().all(): + await session.delete(old) + await session.flush() + + row = ModelMetadata( + model_type=model_type, + version=version, + training_samples=training_samples, + metrics=metrics, + artifact_path=str(artifact_path), + is_active=True, + mlflow_run_id=mlflow_run_id, + threshold=threshold, + ) + session.add(row) + rows.append(row) + + await session.commit() + + logger.info( + "Saved metadata for %d models (samples=%d)", + len(rows), + training_samples, + ) + + return rows diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_metadata.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_metadata.py new file mode 100644 index 00000000..cfb7b7ed --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_metadata.py @@ -0,0 +1,212 @@ +""" +©AngelaMos | 2026 +test_metadata.py +""" + +import json +from pathlib import Path + +import pytest +from sqlalchemy.ext.asyncio import ( + AsyncSession, + async_sessionmaker, + create_async_engine, +) +from sqlalchemy.pool import StaticPool +from sqlalchemy import select +from sqlmodel import SQLModel + +from app.models.model_metadata import ModelMetadata +from ml.metadata import compute_model_version, save_model_metadata + + +@pytest.fixture +async def db_session(tmp_path: Path): + """ + In-memory SQLite session for metadata tests + """ + from app.models import model_metadata as _reg # noqa: F401 + + engine = create_async_engine( + "sqlite+aiosqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + async with engine.begin() as conn: + await conn.run_sync(SQLModel.metadata.create_all) + + factory = async_sessionmaker( + engine, class_=AsyncSession, expire_on_commit=False + ) + async with factory() as session: + yield session + + await engine.dispose() + + +@pytest.fixture +def model_artifacts(tmp_path: Path) -> Path: + """ + Create fake ONNX model files for version hashing + """ + (tmp_path / "ae.onnx").write_bytes(b"ae-model-data-123") + (tmp_path / "rf.onnx").write_bytes(b"rf-model-data-456") + (tmp_path / "if.onnx").write_bytes(b"if-model-data-789") + (tmp_path / "scaler.json").write_text( + json.dumps({"center": [0.0], "scale": [1.0]}) + ) + (tmp_path / "threshold.json").write_text( + json.dumps({"threshold": 0.05}) + ) + return tmp_path + + +class TestComputeModelVersion: + """ + Test SHA-256 based model version hashing + """ + + def test_returns_12_char_hex( + self, model_artifacts: Path + ) -> None: + """ + Version string is a 12-character hex digest + """ + version = compute_model_version( + model_artifacts / "ae.onnx" + ) + + assert len(version) == 12 + assert all(c in "0123456789abcdef" for c in version) + + def test_same_file_same_version( + self, model_artifacts: Path + ) -> None: + """ + Same file produces the same version string + """ + v1 = compute_model_version( + model_artifacts / "ae.onnx" + ) + v2 = compute_model_version( + model_artifacts / "ae.onnx" + ) + + assert v1 == v2 + + def test_different_files_different_versions( + self, model_artifacts: Path + ) -> None: + """ + Different files produce different version strings + """ + v_ae = compute_model_version( + model_artifacts / "ae.onnx" + ) + v_rf = compute_model_version( + model_artifacts / "rf.onnx" + ) + + assert v_ae != v_rf + + +class TestSaveModelMetadata: + """ + Test model metadata persistence to database + """ + + @pytest.mark.asyncio + async def test_creates_three_rows( + self, + db_session: AsyncSession, + model_artifacts: Path, + ) -> None: + """ + save_model_metadata creates one row per model type + """ + rows = await save_model_metadata( + db_session, + model_dir=model_artifacts, + training_samples=500, + metrics={"f1": 0.9, "pr_auc": 0.88}, + ) + + assert len(rows) == 3 + + @pytest.mark.asyncio + async def test_all_rows_active( + self, + db_session: AsyncSession, + model_artifacts: Path, + ) -> None: + """ + All newly saved models are marked as active + """ + rows = await save_model_metadata( + db_session, + model_dir=model_artifacts, + training_samples=500, + metrics={"f1": 0.9}, + ) + + assert all(r.is_active for r in rows) + + @pytest.mark.asyncio + async def test_model_types_correct( + self, + db_session: AsyncSession, + model_artifacts: Path, + ) -> None: + """ + Row model types are autoencoder, random_forest, isolation_forest + """ + rows = await save_model_metadata( + db_session, + model_dir=model_artifacts, + training_samples=500, + metrics={}, + ) + types = {r.model_type for r in rows} + + assert types == { + "autoencoder", + "random_forest", + "isolation_forest", + } + + @pytest.mark.asyncio + async def test_previous_active_replaced( + self, + db_session: AsyncSession, + model_artifacts: Path, + ) -> None: + """ + Saving new metadata replaces previous active models + """ + await save_model_metadata( + db_session, + model_dir=model_artifacts, + training_samples=500, + metrics={"f1": 0.9}, + ) + + (model_artifacts / "ae.onnx").write_bytes( + b"new-ae-data" + ) + await save_model_metadata( + db_session, + model_dir=model_artifacts, + training_samples=600, + metrics={"f1": 0.95}, + ) + + result = await db_session.execute( + select(ModelMetadata) + ) + all_rows = result.scalars().all() + active_rows = [r for r in all_rows if r.is_active] + + assert len(active_rows) == 3 + assert all( + r.training_samples == 600 for r in active_rows + ) From f9f8e1ef5237cfa7e851764b1b393b0a305cf17e Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Sat, 28 Feb 2026 06:29:07 -0500 Subject: [PATCH 05/19] feat(ml): add stratified splitting with SMOTE oversampling 70/15/15 train/val/test split with SMOTE applied only to the training set. Extracts X_normal_train pre-SMOTE for autoencoder. --- .../backend/ml/splitting.py | 86 ++++++++ .../backend/tests/test_splitting.py | 190 ++++++++++++++++++ 2 files changed, 276 insertions(+) create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/ml/splitting.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/tests/test_splitting.py diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/splitting.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/splitting.py new file mode 100644 index 00000000..89c91a91 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/splitting.py @@ -0,0 +1,86 @@ +""" +©AngelaMos | 2026 +splitting.py +""" + +from dataclasses import dataclass + +import numpy as np +from imblearn.over_sampling import SMOTE +from sklearn.model_selection import train_test_split + + +@dataclass +class TrainingSplit: + """ + Result of stratified splitting with SMOTE oversampling + """ + + X_train: np.ndarray + y_train: np.ndarray + X_val: np.ndarray + y_val: np.ndarray + X_test: np.ndarray + y_test: np.ndarray + X_normal_train: np.ndarray + + +def prepare_training_data( + X: np.ndarray, + y: np.ndarray, + train_ratio: float = 0.70, + val_ratio: float = 0.15, + smote_strategy: float = 0.3, + smote_k: int = 5, + random_state: int = 42, +) -> TrainingSplit: + """ + Split data into train/val/test with SMOTE on training set only + """ + n_classes = len(np.unique(y)) + if n_classes < 2: + raise ValueError( + "y must contain at least 2 classes" + ) + + test_size = 1.0 - train_ratio + X_train, X_rem, y_train, y_rem = train_test_split( + X, + y, + test_size=test_size, + stratify=y, + random_state=random_state, + ) + + X_val, X_test, y_val, y_test = train_test_split( + X_rem, + y_rem, + test_size=0.5, + stratify=y_rem, + random_state=random_state, + ) + + X_normal_train = X_train[y_train == 0] + + class_counts = np.bincount(y_train) + minority_count = class_counts.min() + + if minority_count >= smote_k + 1: + sampler = SMOTE( + sampling_strategy=smote_strategy, + k_neighbors=smote_k, + random_state=random_state, + ) + X_train, y_train = sampler.fit_resample( + X_train, y_train + ) + + return TrainingSplit( + X_train=X_train, + y_train=y_train, + X_val=X_val, + y_val=y_val, + X_test=X_test, + y_test=y_test, + X_normal_train=X_normal_train, + ) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_splitting.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_splitting.py new file mode 100644 index 00000000..b907a7b4 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_splitting.py @@ -0,0 +1,190 @@ +""" +©AngelaMos | 2026 +test_splitting.py +""" + +import numpy as np +import pytest + +from ml.splitting import TrainingSplit, prepare_training_data + +N_NORMAL = 160 +N_ATTACK = 40 +N_FEATURES = 35 +TOTAL = N_NORMAL + N_ATTACK + + +def _make_dataset( + n_normal: int = N_NORMAL, + n_attack: int = N_ATTACK, + n_features: int = N_FEATURES, +) -> tuple[np.ndarray, np.ndarray]: + """ + Generate synthetic imbalanced dataset + """ + rng = np.random.default_rng(42) + X = rng.standard_normal( + (n_normal + n_attack, n_features) + ).astype(np.float32) + y = np.array( + [0] * n_normal + [1] * n_attack, + dtype=np.int32, + ) + return X, y + + +class TestPrepareTrainingData: + """ + Tests for stratified splitting with SMOTE + """ + + def test_returns_training_split(self) -> None: + """ + Returns a TrainingSplit dataclass + """ + X, y = _make_dataset() + result = prepare_training_data(X, y) + assert isinstance(result, TrainingSplit) + + def test_split_proportions(self) -> None: + """ + Validates 70/15/15 split proportions + """ + X, y = _make_dataset() + result = prepare_training_data(X, y) + expected_val = int(TOTAL * 0.15) + expected_test = int(TOTAL * 0.15) + tolerance = int(TOTAL * 0.05) + assert abs( + result.X_val.shape[0] - expected_val + ) <= tolerance + assert abs( + result.X_test.shape[0] - expected_test + ) <= tolerance + + def test_stratified_class_distribution( + self, + ) -> None: + """ + Splits preserve the original class ratio + """ + X, y = _make_dataset() + original_ratio = np.mean(y == 1) + result = prepare_training_data(X, y) + val_ratio = np.mean(result.y_val == 1) + test_ratio = np.mean(result.y_test == 1) + ratio_tol = 0.10 + assert abs( + val_ratio - original_ratio + ) < ratio_tol + assert abs( + test_ratio - original_ratio + ) < ratio_tol + + def test_smote_increases_minority(self) -> None: + """ + SMOTE brings minority ratio near strategy + """ + X, y = _make_dataset() + result = prepare_training_data( + X, + y, + smote_strategy=0.3, + ) + minority = np.sum(result.y_train == 1) + majority = np.sum(result.y_train == 0) + ratio = minority / majority + assert ratio >= 0.25 + + def test_val_test_untouched(self) -> None: + """ + Val test sizes match pre-SMOTE counts + """ + X, y = _make_dataset() + result = prepare_training_data(X, y) + remainder = TOTAL - int(TOTAL * 0.70) + half = remainder // 2 + tol = 3 + assert abs( + result.X_val.shape[0] - half + ) <= tol + assert abs( + result.X_test.shape[0] - half + ) <= tol + assert ( + result.X_val.shape[0] + == result.y_val.shape[0] + ) + assert ( + result.X_test.shape[0] + == result.y_test.shape[0] + ) + + def test_x_normal_train_only_normals( + self, + ) -> None: + """ + X_normal_train has only class-0 rows + """ + X, y = _make_dataset() + result = prepare_training_data(X, y) + expected = int(N_NORMAL * 0.70) + tol = int(TOTAL * 0.05) + assert abs( + result.X_normal_train.shape[0] - expected + ) <= tol + assert ( + result.X_normal_train.shape[1] + == N_FEATURES + ) + + def test_small_dataset_works(self) -> None: + """ + Small 50-sample dataset succeeds + """ + X, y = _make_dataset( + n_normal=40, + n_attack=10, + n_features=N_FEATURES, + ) + result = prepare_training_data( + X, y, smote_k=3 + ) + assert isinstance(result, TrainingSplit) + assert result.X_train.shape[0] > 0 + assert result.X_val.shape[0] > 0 + assert result.X_test.shape[0] > 0 + + def test_all_one_class_raises_value_error( + self, + ) -> None: + """ + Single-class labels raise ValueError + """ + rng = np.random.default_rng(99) + X = rng.standard_normal( + (100, N_FEATURES) + ).astype(np.float32) + y_zeros = np.zeros(100, dtype=np.int32) + with pytest.raises(ValueError): + prepare_training_data(X, y_zeros) + y_ones = np.ones(100, dtype=np.int32) + with pytest.raises(ValueError): + prepare_training_data(X, y_ones) + + def test_smote_skipped_minority_too_small( + self, + ) -> None: + """ + Tiny minority skips SMOTE gracefully + """ + X, y = _make_dataset( + n_normal=90, + n_attack=10, + n_features=N_FEATURES, + ) + result = prepare_training_data( + X, y, smote_k=50 + ) + assert isinstance(result, TrainingSplit) + assert result.X_train.shape[0] > 0 From 24eefc5fb4f6df7503c3bd04794f604e591b169a Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Sat, 28 Feb 2026 06:31:22 -0500 Subject: [PATCH 06/19] feat(ml): add synthetic attack and normal traffic generator Six attack generators (SQLi, XSS, traversal, Log4Shell, SSRF, scanner) plus normal traffic, all producing 35-dim feature vectors. --- .../backend/ml/synthetic.py | 509 ++++++++++++++++++ .../backend/tests/test_synthetic.py | 229 ++++++++ 2 files changed, 738 insertions(+) create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/ml/synthetic.py create mode 100644 PROJECTS/advanced/ai-threat-detection/backend/tests/test_synthetic.py diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/synthetic.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/synthetic.py new file mode 100644 index 00000000..cf222bd4 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/synthetic.py @@ -0,0 +1,509 @@ +""" +©AngelaMos | 2026 +synthetic.py +""" + +import logging +import random +from datetime import UTC, datetime, timedelta + +import numpy as np + +from app.core.features.encoder import encode_for_inference +from app.core.features.extractor import extract_request_features +from app.core.ingestion.parsers import ParsedLogEntry + +logger = logging.getLogger(__name__) + +SQLI_PAYLOADS: list[str] = [ + "' OR 1=1--", + "' OR '1'='1", + "' UNION SELECT NULL,NULL--", + "' UNION SELECT username,password FROM users--", + "1; DROP TABLE users--", + "admin'--", + "' AND 1=1--", + "' AND SLEEP(5)--", + "' OR BENCHMARK(1000000,SHA1('test'))--", + "1' ORDER BY 1--", + "1' ORDER BY 10--", + "' UNION ALL SELECT @@version--", + "-1' UNION SELECT 1,CONCAT(user(),database())--", + "' OR 'x'='x", + "1; WAITFOR DELAY '0:0:5'--", + "' AND EXTRACTVALUE(1,CONCAT(0x7e,version()))--", + "' AND UPDATEXML(1,CONCAT(0x7e,version()),1)--", + "admin' AND '1'='1", + "' UNION SELECT LOAD_FILE('/etc/passwd')--", + "' INTO OUTFILE '/tmp/shell.php'--", + "1' AND 1=1 UNION SELECT 1,2,3--", + "' OR EXISTS(SELECT * FROM users)--", +] + +XSS_PAYLOADS: list[str] = [ + "", + "", + "", + "", + "", + "javascript:alert(1)", + "