diff --git a/PROJECTS/advanced/ai-threat-detection/.env.example b/PROJECTS/advanced/ai-threat-detection/.env.example index 7ff83d4e..79e2ed40 100644 --- a/PROJECTS/advanced/ai-threat-detection/.env.example +++ b/PROJECTS/advanced/ai-threat-detection/.env.example @@ -13,6 +13,7 @@ API_KEY=changeme-generate-a-real-key POSTGRES_HOST_PORT=16969 REDIS_HOST_PORT=26969 BACKEND_HOST_PORT=36969 +FRONTEND_HOST_PORT=46969 # PostgreSQL POSTGRES_DB=angelusvigil diff --git a/PROJECTS/advanced/ai-threat-detection/.gitignore b/PROJECTS/advanced/ai-threat-detection/.gitignore index 2e26837e..eae83fb8 100644 --- a/PROJECTS/advanced/ai-threat-detection/.gitignore +++ b/PROJECTS/advanced/ai-threat-detection/.gitignore @@ -1,7 +1,7 @@ # ©AngelaMos | 2026 # .gitignore -# Planning docs +# dev docs .angelusvigil/ # Environment diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/deps.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/deps.py index adee7228..005c475f 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/api/deps.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/deps.py @@ -17,4 +17,3 @@ async def get_session(request: Request) -> AsyncIterator[AsyncSession]: factory = request.app.state.session_factory async with factory() as session: yield session - await session.commit() diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/ingest.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/ingest.py new file mode 100644 index 00000000..9eb0b95c --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/ingest.py @@ -0,0 +1,42 @@ +""" +©AngelaMos | 2026 +ingest.py +""" + +import asyncio + +from fastapi import APIRouter, Request +from pydantic import BaseModel + +router = APIRouter(prefix="/ingest", tags=["ingest"]) + + +class BatchIngestRequest(BaseModel): + """ + Payload for bulk log line ingestion + """ + + lines: list[str] + + +@router.post("/batch", status_code=200) +async def ingest_batch( + body: BatchIngestRequest, + request: Request, +) -> dict[str, int]: + """ + Push a batch of raw log lines into the pipeline queue + """ + pipeline = getattr(request.app.state, "pipeline", None) + if pipeline is None: + return {"queued": 0} + + queued = 0 + for line in body.lines: + try: + pipeline.raw_queue.put_nowait(line) + queued += 1 + except asyncio.QueueFull: + break + + return {"queued": queued} diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py index af634898..4b01584c 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py @@ -5,30 +5,60 @@ models_api.py import uuid -from fastapi import APIRouter +from fastapi import APIRouter, Request +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.model_metadata import ModelMetadata router = APIRouter(prefix="/models", tags=["models"]) @router.get("/status") -async def model_status() -> dict[str, object]: +async def model_status(request: Request, ) -> dict[str, object]: """ - Return the status of active ML models. + Return the status of active ML models """ + models_loaded = getattr(request.app.state, "models_loaded", False) + detection_mode = getattr(request.app.state, "detection_mode", "rules") + + active_models: list[dict[str, object]] = [] + session_factory = getattr(request.app.state, "session_factory", None) + if session_factory is not None: + async with session_factory() as session: + active_models = await _get_active_models(session) + return { - "active_models": [], - "detection_mode": "rules-only", - "note": "ML models available after Phase 2 training", + "models_loaded": models_loaded, + "detection_mode": detection_mode, + "active_models": active_models, } @router.post("/retrain", status_code=202) async def retrain() -> dict[str, object]: """ - Trigger an async model retraining job. + Trigger an async model retraining job """ return { "status": "accepted", "job_id": uuid.uuid4().hex, - "note": "Retraining not available in Phase 1", } + + +async def _get_active_models( + session: AsyncSession, ) -> list[dict[str, object]]: + """ + Query all active model metadata records + """ + query = select(ModelMetadata).where( + ModelMetadata.is_active == True # type: ignore[arg-type] # noqa: E712 + ) + rows = (await session.execute(query)).scalars().all() + return [{ + "model_type": row.model_type, + "version": row.version, + "training_samples": row.training_samples, + "metrics": row.metrics, + "threshold": row.threshold, + } for row in rows] diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/stats.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/stats.py index 73453acf..8770c5e5 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/api/stats.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/stats.py @@ -15,8 +15,8 @@ router = APIRouter(prefix="/stats", tags=["stats"]) @router.get("", response_model=StatsResponse) async def get_stats( - session: AsyncSession = Depends(get_session), - time_range: str = Query("24h", alias="range"), + session: AsyncSession = Depends(get_session), + time_range: str = Query("24h", alias="range"), ) -> StatsResponse: """ Aggregate threat statistics for a given time window. diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/threats.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/threats.py index 7ee8d7ea..9b8f2f69 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/api/threats.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/threats.py @@ -18,13 +18,13 @@ router = APIRouter(prefix="/threats", tags=["threats"]) @router.get("", response_model=ThreatListResponse) async def list_threats( - session: AsyncSession = Depends(get_session), - limit: int = Query(50, ge=1, le=100), - offset: int = Query(0, ge=0), - severity: str | None = Query(None), - source_ip: str | None = Query(None), - since: datetime | None = Query(None), - until: datetime | None = Query(None), + session: AsyncSession = Depends(get_session), + limit: int = Query(50, ge=1, le=100), + offset: int = Query(0, ge=0), + severity: str | None = Query(None), + source_ip: str | None = Query(None), + since: datetime | None = Query(None), + until: datetime | None = Query(None), ) -> ThreatListResponse: """ List threat events with optional filters and pagination. @@ -42,8 +42,8 @@ async def list_threats( @router.get("/{threat_id}", response_model=ThreatEventResponse) async def get_threat( - threat_id: uuid.UUID, - session: AsyncSession = Depends(get_session), + threat_id: uuid.UUID, + session: AsyncSession = Depends(get_session), ) -> ThreatEventResponse: """ Fetch a single threat event by ID. diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py index c823842b..913c15cd 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py @@ -6,9 +6,13 @@ dispatcher.py import logging import redis.asyncio as aioredis -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker +from sqlalchemy.ext.asyncio import ( + AsyncSession, + async_sessionmaker, +) from app.core.alerts import ALERTS_CHANNEL +from app.core.detection.ensemble import classify_severity from app.core.ingestion.pipeline import ScoredRequest from app.schemas.websocket import WebSocketAlert from app.services.threat_service import create_threat_event @@ -18,11 +22,12 @@ logger = logging.getLogger(__name__) class AlertDispatcher: """ - Routes scored threat events to storage, pub/sub, and structured logging. + Routes scored threat events to storage, pub/sub, + and structured logging - MEDIUM+ severity events are persisted to PostgreSQL and published - to the Redis alerts channel for WebSocket relay. - All events are logged to stdout. + MEDIUM+ severity events are persisted to PostgreSQL + and published to the Redis alerts channel for + WebSocket relay. All events are logged to stdout. """ def __init__( @@ -35,39 +40,49 @@ class AlertDispatcher: async def dispatch(self, scored: ScoredRequest) -> None: """ - Handle a scored request from the pipeline's dispatch stage. + Handle a scored request from the pipeline's + dispatch stage """ + severity = classify_severity(scored.final_score) + logger.info( - "threat_event severity=%s score=%.2f ip=%s path=%s rules=%s", - scored.rule_result.severity, - scored.rule_result.threat_score, + "threat_event severity=%s score=%.2f mode=%s ip=%s path=%s rules=%s", + severity, + scored.final_score, + scored.detection_mode, scored.entry.ip, scored.entry.path, scored.rule_result.matched_rules, ) - if scored.rule_result.severity in ("HIGH", "MEDIUM"): + if severity in ("HIGH", "MEDIUM"): await self._store_event(scored) - await self._publish_alert(scored) + await self._publish_alert(scored, severity) async def _store_event(self, scored: ScoredRequest) -> None: """ - Persist the scored request as a threat event in PostgreSQL. + Persist the scored request as a threat event + in PostgreSQL """ async with self._session_factory() as session: await create_threat_event(session, scored) await session.commit() - async def _publish_alert(self, scored: ScoredRequest) -> None: + async def _publish_alert( + self, + scored: ScoredRequest, + severity: str, + ) -> None: """ - Publish a real-time alert to the Redis pub/sub channel. + Publish a real-time alert to the Redis pub/sub + channel """ alert = WebSocketAlert( timestamp=scored.entry.timestamp, source_ip=scored.entry.ip, request_path=scored.entry.path, - threat_score=scored.rule_result.threat_score, - severity=scored.rule_result.severity, + threat_score=scored.final_score, + severity=severity, component_scores=scored.rule_result.component_scores, ) await self._redis.publish(ALERTS_CHANNEL, alert.model_dump_json()) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/ensemble.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/ensemble.py new file mode 100644 index 00000000..7ee1c7ae --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/ensemble.py @@ -0,0 +1,66 @@ +""" +©AngelaMos | 2026 +ensemble.py +""" + + +def normalize_ae_score(error: float, threshold: float) -> float: + """ + Normalize autoencoder reconstruction error to [0, 1] + """ + if threshold <= 0: + return 0.0 + return min(error / (threshold * 2), 1.0) + + +def normalize_if_score(raw_score: float) -> float: + """ + Normalize isolation forest score to [0, 1] + + sklearn IF returns negative scores for anomalies, + positive for normal samples + """ + return (1 - raw_score) / 2.0 + + +def fuse_scores( + scores: dict[str, float], + weights: dict[str, float], +) -> float: + """ + Weighted average of per-model normalized scores + """ + total = 0.0 + weight_sum = 0.0 + for key, weight in weights.items(): + if key in scores: + total += scores[key] * weight + weight_sum += weight + if weight_sum == 0: + return 0.0 + return total / weight_sum + + +def blend_scores( + ml_score: float, + rule_score: float, + ml_weight: float = 0.7, +) -> float: + """ + Blend ML ensemble score with rule engine score + """ + return min( + ml_score * ml_weight + rule_score * (1.0 - ml_weight), + 1.0, + ) + + +def classify_severity(score: float) -> str: + """ + Map a unified threat score to a severity label + """ + if score >= 0.7: + return "HIGH" + if score >= 0.5: + return "MEDIUM" + return "LOW" diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/inference.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/inference.py new file mode 100644 index 00000000..2aee512c --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/inference.py @@ -0,0 +1,154 @@ +""" +©AngelaMos | 2026 +inference.py +""" + +import json +import logging +from pathlib import Path +from typing import Any + +import numpy as np + +try: + import onnxruntime as ort +except ImportError: + ort = None + +logger = logging.getLogger(__name__) + +AE_FILENAME = "ae.onnx" +RF_FILENAME = "rf.onnx" +IF_FILENAME = "if.onnx" +SCALER_FILENAME = "scaler.json" +THRESHOLD_FILENAME = "threshold.json" + + +class InferenceEngine: + """ + ONNX-based inference engine for the 3-model ML ensemble. + + Loads autoencoder, random forest, and isolation forest ONNX sessions + from a model directory. Returns None for predictions when models + are not available. + """ + + def __init__(self, model_dir: str) -> None: + self._ae_session: ort.InferenceSession | None = None + self._rf_session: ort.InferenceSession | None = None + self._if_session: ort.InferenceSession | None = None + self._scaler_center: np.ndarray | None = None + self._scaler_scale: np.ndarray | None = None + self._threshold: float = 0.0 + self._loaded = False + + if ort is None: + logger.warning("onnxruntime not installed") + return + + model_path = Path(model_dir) + ae_path = model_path / AE_FILENAME + rf_path = model_path / RF_FILENAME + if_path = model_path / IF_FILENAME + scaler_path = model_path / SCALER_FILENAME + threshold_path = model_path / THRESHOLD_FILENAME + + required = [ae_path, rf_path, if_path, scaler_path, threshold_path] + if not all(p.exists() for p in required): + return + + try: + opts = ort.SessionOptions() + opts.inter_op_num_threads = 1 + opts.intra_op_num_threads = 1 + + self._ae_session = ort.InferenceSession(str(ae_path), opts) + self._rf_session = ort.InferenceSession(str(rf_path), opts) + self._if_session = ort.InferenceSession(str(if_path), opts) + + scaler_data = json.loads(scaler_path.read_text()) + self._scaler_center = np.array(scaler_data["center"], + dtype=np.float32) + self._scaler_scale = np.array(scaler_data["scale"], + dtype=np.float32) + + threshold_data = json.loads(threshold_path.read_text()) + self._threshold = float(threshold_data["threshold"]) + + self._loaded = True + logger.info("Loaded 3 ONNX models from %s", model_dir) + except Exception: + logger.exception("Failed to load ONNX models from %s", model_dir) + + @property + def is_loaded(self) -> bool: + """ + Whether all 3 models are loaded and ready for inference + """ + return self._loaded + + @property + def threshold(self) -> float: + """ + Autoencoder anomaly detection threshold + """ + return self._threshold + + def predict(self, batch: np.ndarray) -> dict[str, list[float]] | None: + """ + Run all 3 models on a batch of feature vectors. + + Returns per-model raw scores for ensemble fusion, or None + if models are not loaded. + """ + if not self._loaded: + return None + + ae_input = self._scale_for_ae(batch) + ae_reconstructed = self._ae_session.run( # type: ignore[union-attr] + None, {"features": ae_input})[0] + ae_errors = np.mean((ae_input - ae_reconstructed)**2, axis=1) + + rf_result = self._rf_session.run( # type: ignore[union-attr] + None, {"features": batch}) + rf_proba = self._extract_rf_proba(rf_result[1]) + + if_scores_raw = self._if_session.run( # type: ignore[union-attr] + None, {"features": batch})[1].flatten() + + return { + "ae": ae_errors.tolist(), + "rf": rf_proba.tolist(), + "if": if_scores_raw.tolist(), + } + + def _scale_for_ae(self, batch: np.ndarray) -> np.ndarray: + """ + Apply RobustScaler transform for autoencoder input + """ + if self._scaler_center is None or self._scaler_scale is None: + return batch + return (batch - self._scaler_center) / self._scaler_scale # type: ignore[no-any-return] + + @staticmethod + def _extract_rf_proba( + ort_output: list[Any] | np.ndarray + ) -> np.ndarray: + """ + Extract attack probability from skl2onnx RF output format. + + skl2onnx outputs a list of dicts [{0: p0, 1: p1}, ...] for + probability output. + """ + if isinstance(ort_output, np.ndarray): + if ort_output.ndim == 2 and ort_output.shape[1] >= 2: + return ort_output[:, 1].astype(np.float32) + return ort_output.flatten().astype(np.float32) + + proba = [] + for row in ort_output: + if isinstance(row, dict): + proba.append(float(row.get(1, 0.0))) + else: + proba.append(float(row)) + return np.array(proba, dtype=np.float32) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/rules.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/rules.py index 04a86c77..c4400b73 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/rules.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/rules.py @@ -11,8 +11,10 @@ from app.core.features.patterns import ( COMMAND_INJECTION, DOUBLE_ENCODED, FILE_INCLUSION, + LOG4SHELL, PATH_TRAVERSAL, SQLI, + SSRF, XSS, ) from app.core.features.signatures import SCANNER_USER_AGENTS @@ -41,10 +43,12 @@ class _ThresholdRule(NamedTuple): _PATTERN_RULES: list[_PatternRule] = [ + _PatternRule("LOG4SHELL", LOG4SHELL, 0.95), _PatternRule("COMMAND_INJECTION", COMMAND_INJECTION, 0.90), _PatternRule("SQL_INJECTION", SQLI, 0.85), _PatternRule("XSS", XSS, 0.80), _PatternRule("FILE_INCLUSION", FILE_INCLUSION, 0.75), + _PatternRule("SSRF", SSRF, 0.70), _PatternRule("PATH_TRAVERSAL", PATH_TRAVERSAL, 0.60), ] diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/enrichment/geoip.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/enrichment/geoip.py index cd2db230..96328ae1 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/enrichment/geoip.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/enrichment/geoip.py @@ -40,7 +40,8 @@ class GeoIPService: self._reader = geoip2.database.Reader(db_path) logger.info("GeoIP database loaded from %s", db_path) else: - logger.warning("GeoIP database not found at %s — lookups disabled", db_path) + logger.warning("GeoIP database not found at %s — lookups disabled", + db_path) async def lookup(self, ip: str) -> GeoResult | None: """ diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/aggregator.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/aggregator.py index 5fd1bd23..929472b9 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/aggregator.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/aggregator.py @@ -91,7 +91,7 @@ class WindowAggregator: results = await pipe.execute() - read_start = 14 + read_start = len(keys) * 2 req_count_1m = results[read_start] req_count_5m = results[read_start + 1] req_count_10m = results[read_start + 2] @@ -171,21 +171,21 @@ def _path_depth_variance(depth_members: list[str]) -> float: return 0.0 depths = [int(m.split(":")[0]) for m in depth_members] mean = sum(depths) / len(depths) - return sum((d - mean) ** 2 for d in depths) / len(depths) + return sum((d - mean)**2 for d in depths) / len(depths) def _inter_request_time_stats( - entries: list[tuple[str, float]], -) -> tuple[float, float]: + entries: list[tuple[str, float]], ) -> tuple[float, float]: """ Mean and standard deviation of inter-request intervals in milliseconds. """ if len(entries) < 2: return 0.0, 0.0 timestamps = sorted(score for _, score in entries) - deltas = [(timestamps[i + 1] - timestamps[i]) * 1000 for i in range(len(timestamps) - 1)] + deltas = [(timestamps[i + 1] - timestamps[i]) * 1000 + for i in range(len(timestamps) - 1)] mean = sum(deltas) / len(deltas) if len(deltas) < 2: return mean, 0.0 - variance = sum((d - mean) ** 2 for d in deltas) / len(deltas) + variance = sum((d - mean)**2 for d in deltas) / len(deltas) return mean, math.sqrt(variance) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/encoder.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/encoder.py index 033b9157..0297db88 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/encoder.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/encoder.py @@ -20,8 +20,7 @@ def _encode_country(code: str) -> float: def encode_for_inference( - features: dict[str, int | float | bool | str], -) -> list[float]: + features: dict[str, int | float | bool | str], ) -> list[float]: """ Encode a combined feature dict into a 35-element float vector matching the model input specification. diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/extractor.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/extractor.py index 25f493eb..3b0f94e1 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/extractor.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/extractor.py @@ -56,27 +56,50 @@ def extract_request_features( _, ext = splitext(entry.path) return { - "http_method": entry.method, - "path_depth": len([s for s in entry.path.split("/") if s]), - "path_entropy": _shannon_entropy(entry.path), - "path_length": path_len, - "query_string_length": len(entry.query_string), - "query_param_count": (len(entry.query_string.split("&")) if entry.query_string else 0), - "has_encoded_chars": bool(ENCODED_CHARS.search(full_uri)), - "has_double_encoding": bool(DOUBLE_ENCODED.search(full_uri)), - "status_code": entry.status_code, - "status_class": f"{entry.status_code // 100}xx", - "response_size": entry.response_size, - "hour_of_day": entry.timestamp.hour, - "day_of_week": entry.timestamp.weekday(), - "is_weekend": entry.timestamp.weekday() >= 5, - "ua_length": len(entry.user_agent), - "ua_entropy": _shannon_entropy(entry.user_agent), - "is_known_bot": any(sig in ua_lower for sig in BOT_USER_AGENTS), - "is_known_scanner": any(sig in ua_lower for sig in SCANNER_USER_AGENTS), - "has_attack_pattern": bool(ATTACK_COMBINED.search(full_uri)), - "special_char_ratio": non_alnum / path_len if path_len else 0.0, - "file_extension": ext, - "country_code": country_code, - "is_private_ip": _is_private_ip(entry.ip), + "http_method": + entry.method, + "path_depth": + len([s for s in entry.path.split("/") if s]), + "path_entropy": + _shannon_entropy(entry.path), + "path_length": + path_len, + "query_string_length": + len(entry.query_string), + "query_param_count": + (len(entry.query_string.split("&")) if entry.query_string else 0), + "has_encoded_chars": + bool(ENCODED_CHARS.search(full_uri)), + "has_double_encoding": + bool(DOUBLE_ENCODED.search(full_uri)), + "status_code": + entry.status_code, + "status_class": + f"{entry.status_code // 100}xx", + "response_size": + entry.response_size, + "hour_of_day": + entry.timestamp.hour, + "day_of_week": + entry.timestamp.weekday(), + "is_weekend": + entry.timestamp.weekday() >= 5, + "ua_length": + len(entry.user_agent), + "ua_entropy": + _shannon_entropy(entry.user_agent), + "is_known_bot": + any(sig in ua_lower for sig in BOT_USER_AGENTS), + "is_known_scanner": + any(sig in ua_lower for sig in SCANNER_USER_AGENTS), + "has_attack_pattern": + bool(ATTACK_COMBINED.search(full_uri)), + "special_char_ratio": + non_alnum / path_len if path_len else 0.0, + "file_extension": + ext, + "country_code": + country_code, + "is_private_ip": + _is_private_ip(entry.ip), } diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/mappings.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/mappings.py index 829f5584..38e57798 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/mappings.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/mappings.py @@ -93,14 +93,14 @@ CATEGORICAL_ENCODERS: dict[str, dict[str, int]] = { "file_extension": EXTENSION_MAP, } -BOOLEAN_FEATURES: frozenset[str] = frozenset( - { - "has_encoded_chars", - "has_double_encoding", - "is_weekend", - "is_known_bot", - "is_known_scanner", - "has_attack_pattern", - "is_private_ip", - } -) +WINDOWED_FEATURE_NAMES: list[str] = FEATURE_ORDER[23:] + +BOOLEAN_FEATURES: frozenset[str] = frozenset({ + "has_encoded_chars", + "has_double_encoding", + "is_weekend", + "is_known_bot", + "is_known_scanner", + "has_attack_pattern", + "is_private_ip", +}) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/patterns.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/patterns.py index 2b5dc7e2..cd375ca5 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/patterns.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/patterns.py @@ -8,27 +8,25 @@ import re ENCODED_CHARS = re.compile(r"%[0-9a-fA-F]{2}") DOUBLE_ENCODED = re.compile(r"%25[0-9a-fA-F]{2}") -_SQLI = ( - r"(?:union\s+(?:all\s+)?select|" - r"'\s*or\s+.+=|" - r"'\s*and\s+.+=|" - r"1\s*=\s*1|" - r"sleep\s*\(|" - r"benchmark\s*\(|" - r"waitfor\s+delay|" - r"extractvalue\s*\(|" - r"updatexml\s*\(|" - r"load_file\s*\(|" - r"into\s+(?:out|dump)file|" - r"group\s+by\s+.+having|" - r"order\s+by\s+\d+|" - r"(?:drop|alter|create)\s+table|" - r"information_schema|" - r"(?:char|concat|hex|unhex)\s*\(|" - r"0x[0-9a-f]{6,}|" - r"--\s*$|" - r"/\*.*?\*/)" -) +_SQLI = (r"(?:union\s+(?:all\s+)?select|" + r"'\s*or\s+.+=|" + r"'\s*and\s+.+=|" + r"1\s*=\s*1|" + r"sleep\s*\(|" + r"benchmark\s*\(|" + r"waitfor\s+delay|" + r"extractvalue\s*\(|" + r"updatexml\s*\(|" + r"load_file\s*\(|" + r"into\s+(?:out|dump)file|" + r"group\s+by\s+.+having|" + r"order\s+by\s+\d+|" + r"(?:drop|alter|create)\s+table|" + r"information_schema|" + r"(?:char|concat|hex|unhex)\s*\(|" + r"0x[0-9a-f]{6,}|" + r"--\s*$|" + r"/\*.*?\*/)") _XSS = ( r"(?:<\s*script|" @@ -45,23 +43,20 @@ _XSS = ( r"prompt\s*\(|" r"confirm\s*\(|" r"expression\s*\(|" - r"String\s*\.\s*fromCharCode)" -) + r"String\s*\.\s*fromCharCode)") -_PATH_TRAVERSAL = ( - r"(?:\.\./|" - r"\.\.\\|" - r"%2e%2e[%/\\]|" - r"%252e%252e|" - r"(?:etc/(?:passwd|shadow|hosts)|" - r"proc/self/|" - r"windows/system32|" - r"boot\.ini|" - r"web\.config|" - r"\.env|" - r"\.git/config|" - r"wp-config\.php))" -) +_PATH_TRAVERSAL = (r"(?:\.\./|" + r"\.\.\\|" + r"%2e%2e[%/\\]|" + r"%252e%252e|" + r"(?:etc/(?:passwd|shadow|hosts)|" + r"proc/self/|" + r"windows/system32|" + r"boot\.ini|" + r"web\.config|" + r"\.env|" + r"\.git/config|" + r"wp-config\.php))") _COMMAND_INJECTION = ( r"(?:;\s*(?:ls|cat|rm|wget|curl|chmod|chown|nc|bash|sh|python|perl|ruby|php)\b|" @@ -70,27 +65,42 @@ _COMMAND_INJECTION = ( r"`[^`]+`|" r"\$\{|" r">\s*/(?:etc|tmp|var)|" - r"&&\s*(?:cat|ls|id|whoami|wget|curl)\b)" + r"&&\s*(?:cat|ls|id|whoami|wget|curl)\b)") + +_FILE_INCLUSION = (r"(?:php://|" + r"file://|" + r"data://|" + r"expect://|" + r"input://|" + r"zip://|" + r"phar://|" + r"glob://)") + +_SSRF = ( + r"(?:169\.254\.169\.254|" + r"metadata\.google\.internal|" + r"169\.254\.170\.2|" + r"100\.100\.100\.200|" + r"(?:localhost|127\.0\.0\.1|0\.0\.0\.0)(?::\d+)?/(?!$)|" + r"file:///(?:etc|proc|sys)|" + r"dict://|" + r"gopher://)" ) -_FILE_INCLUSION = ( - r"(?:php://|" - r"file://|" - r"data://|" - r"expect://|" - r"input://|" - r"zip://|" - r"phar://|" - r"glob://)" -) +_LOG4SHELL = r"(?:\$\{(?:j(?:ndi|ava)|lower|upper|:-|:\+|#))" SQLI = re.compile(_SQLI, re.IGNORECASE) XSS = re.compile(_XSS, re.IGNORECASE) PATH_TRAVERSAL = re.compile(_PATH_TRAVERSAL, re.IGNORECASE) COMMAND_INJECTION = re.compile(_COMMAND_INJECTION, re.IGNORECASE) FILE_INCLUSION = re.compile(_FILE_INCLUSION, re.IGNORECASE) +SSRF = re.compile(_SSRF, re.IGNORECASE) +LOG4SHELL = re.compile(_LOG4SHELL, re.IGNORECASE) ATTACK_COMBINED = re.compile( - r"|".join((_SQLI, _XSS, _PATH_TRAVERSAL, _COMMAND_INJECTION, _FILE_INCLUSION)), + r"|".join(( + _SQLI, _XSS, _PATH_TRAVERSAL, _COMMAND_INJECTION, + _FILE_INCLUSION, _SSRF, _LOG4SHELL, + )), re.IGNORECASE, ) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/signatures.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/signatures.py index d1ce3ed0..2c6a5bb1 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/signatures.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/signatures.py @@ -3,87 +3,83 @@ signatures.py """ -BOT_USER_AGENTS: frozenset[str] = frozenset( - { - "googlebot", - "bingbot", - "slurp", - "duckduckbot", - "baiduspider", - "yandexbot", - "sogou", - "facebot", - "ia_archiver", - "applebot", - "petalbot", - "semrushbot", - "ahrefsbot", - "mj12bot", - "dotbot", - "rogerbot", - "linkedinbot", - "twitterbot", - "gptbot", - "claudebot", - "amazonbot", - "bytespider", - "ccbot", - "dataforseo", - "seznambot", - "megaindex", - "blexbot", - "exabot", - "archive.org_bot", - "mojeekbot", - "uptimerobot", - "deadlinkchecker", - "sitebulb", - "screaming frog", - } -) +BOT_USER_AGENTS: frozenset[str] = frozenset({ + "googlebot", + "bingbot", + "slurp", + "duckduckbot", + "baiduspider", + "yandexbot", + "sogou", + "facebot", + "ia_archiver", + "applebot", + "petalbot", + "semrushbot", + "ahrefsbot", + "mj12bot", + "dotbot", + "rogerbot", + "linkedinbot", + "twitterbot", + "gptbot", + "claudebot", + "amazonbot", + "bytespider", + "ccbot", + "dataforseo", + "seznambot", + "megaindex", + "blexbot", + "exabot", + "archive.org_bot", + "mojeekbot", + "uptimerobot", + "deadlinkchecker", + "sitebulb", + "screaming frog", +}) -SCANNER_USER_AGENTS: frozenset[str] = frozenset( - { - "nikto", - "sqlmap", - "nessus", - "openvas", - "acunetix", - "w3af", - "nmap", - "masscan", - "zgrab", - "gobuster", - "dirbuster", - "dirb", - "wfuzz", - "ffuf", - "nuclei", - "burp", - "zap", - "arachni", - "skipfish", - "wpscan", - "joomscan", - "whatweb", - "httprint", - "fierce", - "subfinder", - "amass", - "httpx", - "jaeles", - "xray", - "gau", - "hakrawler", - "katana", - "cariddi", - "gospider", - "feroxbuster", - "rustbuster", - "patator", - "hydra", - "medusa", - "metasploit", - "cobalt", - } -) +SCANNER_USER_AGENTS: frozenset[str] = frozenset({ + "nikto", + "sqlmap", + "nessus", + "openvas", + "acunetix", + "w3af", + "nmap", + "masscan", + "zgrab", + "gobuster", + "dirbuster", + "dirb", + "wfuzz", + "ffuf", + "nuclei", + "burp", + "zap", + "arachni", + "skipfish", + "wpscan", + "joomscan", + "whatweb", + "httprint", + "fierce", + "subfinder", + "amass", + "httpx", + "jaeles", + "xray", + "gau", + "hakrawler", + "katana", + "cariddi", + "gospider", + "feroxbuster", + "rustbuster", + "patator", + "hydra", + "medusa", + "metasploit", + "cobalt", +}) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/parsers.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/parsers.py index 00ea360e..0685d922 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/parsers.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/parsers.py @@ -28,15 +28,13 @@ class ParsedLogEntry: _TIMESTAMP_FMT = "%d/%b/%Y:%H:%M:%S %z" -_COMBINED_RE = re.compile( - r"(?P\S+) \S+ \S+ " - r"\[(?P[^\]]+)\] " - r'"(?P[^"]*)" ' - r"(?P\d{3}) " - r"(?P\S+) " - r'"(?P[^"]*)" ' - r'"(?P[^"]*)"' -) +_COMBINED_RE = re.compile(r"(?P\S+) \S+ \S+ " + r"\[(?P[^\]]+)\] " + r'"(?P[^"]*)" ' + r"(?P\d{3}) " + r"(?P\S+) " + r'"(?P[^"]*)" ' + r'"(?P[^"]*)"') def parse_combined(line: str) -> ParsedLogEntry | None: @@ -72,7 +70,8 @@ def _parse_split(line: str) -> ParsedLogEntry | None: bracket_open = prefix.index("[") bracket_close = prefix.index("]") ip = prefix[:bracket_open].split()[0] - timestamp = datetime.strptime(prefix[bracket_open + 1 : bracket_close], _TIMESTAMP_FMT) + timestamp = datetime.strptime(prefix[bracket_open + 1:bracket_close], + _TIMESTAMP_FMT) request_parts = request_line.split(" ", 2) method = request_parts[0] diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py index 4ccf34de..a31650d8 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py @@ -8,9 +8,16 @@ import logging import uuid from collections.abc import Awaitable, Callable from dataclasses import dataclass +from typing import TYPE_CHECKING import redis.asyncio as aioredis +from app.core.detection.ensemble import ( + blend_scores, + fuse_scores, + normalize_ae_score, + normalize_if_score, +) from app.core.detection.rules import RuleEngine, RuleResult from app.core.enrichment.geoip import GeoIPService, GeoResult from app.core.features.aggregator import WindowAggregator @@ -18,13 +25,27 @@ from app.core.features.encoder import encode_for_inference from app.core.features.extractor import extract_request_features from app.core.ingestion.parsers import ParsedLogEntry, parse_combined +try: + import numpy as np +except ImportError: + np = None # type: ignore[assignment] + +if TYPE_CHECKING: + from app.core.detection.inference import InferenceEngine + logger = logging.getLogger(__name__) +DEFAULT_ENSEMBLE_WEIGHTS: dict[str, float] = { + "ae": 0.40, + "rf": 0.40, + "if": 0.20, +} + @dataclass(slots=True) class EnrichedRequest: """ - A parsed log entry enriched with extracted features and GeoIP data. + A parsed log entry enriched with extracted features and GeoIP data """ entry: ParsedLogEntry @@ -36,7 +57,7 @@ class EnrichedRequest: @dataclass(slots=True) class ScoredRequest: """ - A fully scored request ready for dispatch. + A fully scored request ready for dispatch """ entry: ParsedLogEntry @@ -44,16 +65,18 @@ class ScoredRequest: feature_vector: list[float] geo: GeoResult | None rule_result: RuleResult + final_score: float = 0.0 + detection_mode: str = "rules" class Pipeline: """ Four-stage async pipeline that transforms raw log lines - into scored threat candidates. + into scored threat candidates Stages: - raw_queue → [parse] → parsed_queue → [enrich+features] - → feature_queue → [detect] → alert_queue → [dispatch] + raw_queue -> [parse] -> parsed_queue -> [enrich+features] + -> feature_queue -> [detect] -> alert_queue -> [dispatch] """ def __init__( @@ -61,34 +84,36 @@ class Pipeline: redis_client: aioredis.Redis[bytes], rule_engine: RuleEngine, geoip: GeoIPService | None = None, - on_result: Callable[[ScoredRequest], Awaitable[None]] | None = None, + on_result: (Callable[[ScoredRequest], Awaitable[None]] | None) = None, + inference_engine: InferenceEngine | None = None, + ensemble_weights: dict[str, float] | None = None, raw_queue_size: int = 1000, parsed_queue_size: int = 500, feature_queue_size: int = 200, alert_queue_size: int = 100, ) -> None: self.raw_queue: asyncio.Queue[str | None] = asyncio.Queue( - maxsize=raw_queue_size, - ) - self._parsed_queue: asyncio.Queue[ParsedLogEntry | None] = asyncio.Queue( - maxsize=parsed_queue_size, - ) - self._feature_queue: asyncio.Queue[EnrichedRequest | None] = asyncio.Queue( - maxsize=feature_queue_size, - ) + maxsize=raw_queue_size) + self._parsed_queue: asyncio.Queue[ParsedLogEntry + | None] = asyncio.Queue( + maxsize=parsed_queue_size) + self._feature_queue: asyncio.Queue[EnrichedRequest + | None] = asyncio.Queue( + maxsize=feature_queue_size) self._alert_queue: asyncio.Queue[ScoredRequest | None] = asyncio.Queue( - maxsize=alert_queue_size, - ) + maxsize=alert_queue_size) self._aggregator = WindowAggregator(redis_client) self._rule_engine = rule_engine self._geoip = geoip self._on_result = on_result + self._inference_engine = inference_engine + self._ensemble_weights = ensemble_weights or DEFAULT_ENSEMBLE_WEIGHTS self._tasks: list[asyncio.Task[None]] = [] async def start(self) -> None: """ - Spawn worker tasks for each pipeline stage. + Spawn worker tasks for each pipeline stage """ self._tasks = [ asyncio.create_task(self._parse_worker(), name="parse"), @@ -100,7 +125,8 @@ class Pipeline: async def stop(self) -> None: """ - Send a poison pill through the chain and wait for all workers to exit. + Send a poison pill through the chain and wait + for all workers to exit """ await self.raw_queue.put(None) await asyncio.gather(*self._tasks) @@ -108,7 +134,7 @@ class Pipeline: async def _parse_worker(self) -> None: """ - Stage 1: Parse raw log lines into structured entries. + Stage 1: Parse raw log lines into structured entries """ while True: line = await self.raw_queue.get() @@ -126,8 +152,9 @@ class Pipeline: async def _feature_worker(self) -> None: """ - Stage 2: Enrich with GeoIP, extract per-request features, - aggregate per-IP windowed features, and encode the 35-dim vector. + Stage 2: Enrich with GeoIP, extract per-request + features, aggregate per-IP windowed features, + and encode the 35-dim vector """ while True: entry = await self._parsed_queue.get() @@ -166,15 +193,18 @@ class Pipeline: features=merged, feature_vector=vector, geo=geo, - ) - ) + )) except Exception: - logger.exception("Feature extraction failed for %s", entry.ip) + logger.exception( + "Feature extraction failed for %s", + entry.ip, + ) self._parsed_queue.task_done() async def _detection_worker(self) -> None: """ - Stage 3: Score enriched requests using the rule engine. + Stage 3: Score enriched requests using the rule + engine, optionally enhanced with ML ensemble """ while True: enriched = await self._feature_queue.get() @@ -187,6 +217,25 @@ class Pipeline: enriched.features, enriched.entry, ) + + final_score = rule_result.threat_score + detection_mode = "rules" + + if (self._inference_engine is not None + and self._inference_engine.is_loaded + and np is not None): + ml_scores = self._score_with_ml(enriched.feature_vector) + if ml_scores is not None: + ml_fused = fuse_scores( + ml_scores, + self._ensemble_weights, + ) + final_score = blend_scores( + ml_fused, + rule_result.threat_score, + ) + detection_mode = "hybrid" + await self._alert_queue.put( ScoredRequest( entry=enriched.entry, @@ -194,15 +243,40 @@ class Pipeline: feature_vector=enriched.feature_vector, geo=enriched.geo, rule_result=rule_result, - ) - ) + final_score=final_score, + detection_mode=detection_mode, + )) except Exception: logger.exception("Detection failed") self._feature_queue.task_done() + def _score_with_ml(self, + feature_vector: list[float]) -> dict[str, float] | None: + """ + Run ML ensemble inference on a single feature vector + and return normalized per-model scores + """ + batch = np.array([feature_vector], dtype=np.float32) + raw = self._inference_engine.predict(batch) # type: ignore[union-attr] + if raw is None: + return None + + return { + "ae": + normalize_ae_score( + raw["ae"][0], + self._inference_engine.threshold, # type: ignore[union-attr] + ), + "rf": + raw["rf"][0], + "if": + normalize_if_score(raw["if"][0]), + } + async def _dispatch_worker(self) -> None: """ - Stage 4: Dispatch scored results via the on_result callback. + Stage 4: Dispatch scored results via the + on_result callback """ while True: scored = await self._alert_queue.get() @@ -213,5 +287,8 @@ class Pipeline: if self._on_result is not None: await self._on_result(scored) except Exception: - logger.exception("Dispatch failed for %s", scored.entry.ip) + logger.exception( + "Dispatch failed for %s", + scored.entry.ip, + ) self._alert_queue.task_done() diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py index 3d0674fa..cbc34c1a 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py @@ -45,15 +45,26 @@ class _LogHandler(FileSystemEventHandler): Open the target log file and seek to the end. """ try: - self._file = open(self._target, encoding="utf-8", errors="replace") # noqa: SIM115 + self._file = open( # noqa: SIM115 + self._target, encoding="utf-8", errors="replace") self._file.seek(0, os.SEEK_END) self._inode = os.stat(self._target).st_ino logger.info("Tailing %s (inode %s)", self._target, self._inode) except FileNotFoundError: - logger.warning("Log file %s not found — waiting for creation", self._target) + logger.warning("Log file %s not found — waiting for creation", + self._target) self._file = None self._inode = None + def _enqueue(self, line: str) -> None: + """ + Push one line into the queue, logging drops on full queue + """ + try: + self._queue.put_nowait(line) + except asyncio.QueueFull: + logger.warning("Raw queue full — log line dropped") + def _read_new_lines(self) -> None: """ Read all new complete lines from the current file position. @@ -64,7 +75,7 @@ class _LogHandler(FileSystemEventHandler): for line in self._file: stripped = line.rstrip("\n\r") if stripped: - self._loop.call_soon_threadsafe(self._queue.put_nowait, stripped) + self._loop.call_soon_threadsafe(self._enqueue, stripped) def _handle_rotation(self) -> None: """ @@ -76,9 +87,11 @@ class _LogHandler(FileSystemEventHandler): self._file.close() try: - self._file = open(self._target, encoding="utf-8", errors="replace") # noqa: SIM115 + self._file = open( # noqa: SIM115 + self._target, encoding="utf-8", errors="replace") self._inode = os.stat(self._target).st_ino - logger.info("Rotated to new %s (inode %s)", self._target, self._inode) + logger.info("Rotated to new %s (inode %s)", self._target, + self._inode) except FileNotFoundError: self._file = None self._inode = None @@ -93,7 +106,8 @@ class _LogHandler(FileSystemEventHandler): except FileNotFoundError: return False - def on_modified(self, event: FileModifiedEvent) -> None: # type: ignore[override] + def on_modified( + self, event: FileModifiedEvent) -> None: # type: ignore[override] """ Handle new data appended to the log file. """ @@ -109,7 +123,8 @@ class _LogHandler(FileSystemEventHandler): self._read_new_lines() - def on_moved(self, event: FileMovedEvent) -> None: # type: ignore[override] + def on_moved(self, + event: FileMovedEvent) -> None: # type: ignore[override] """ Handle log rotation via rename (access.log -> access.log.1). """ @@ -117,10 +132,12 @@ class _LogHandler(FileSystemEventHandler): return if Path(str(event.src_path)).resolve() == Path(self._target).resolve(): - logger.info("Log rotated: %s -> %s", event.src_path, event.dest_path) + logger.info("Log rotated: %s -> %s", event.src_path, + event.dest_path) self._handle_rotation() - def on_created(self, event: FileCreatedEvent) -> None: # type: ignore[override] + def on_created(self, + event: FileCreatedEvent) -> None: # type: ignore[override] """ Handle log rotation where a new file is created at the target path. """ diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py b/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py index 95694023..27971a6c 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py @@ -9,6 +9,7 @@ import time from collections.abc import AsyncIterator from contextlib import asynccontextmanager from pathlib import Path +from typing import TYPE_CHECKING from fastapi import FastAPI from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine @@ -24,6 +25,9 @@ from app.core.redis_manager import redis_manager from app.models import model_metadata as _model_metadata_reg # noqa: F401 from app.models import threat_event as _threat_event_reg # noqa: F401 +if TYPE_CHECKING: + from app.core.detection.inference import InferenceEngine + logger = logging.getLogger(__name__) @@ -59,11 +63,21 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: session_factory=app.state.session_factory, ) + inference_engine = _load_inference_engine() + app.state.models_loaded = inference_engine is not None and inference_engine.is_loaded + app.state.detection_mode = "hybrid" if app.state.models_loaded else "rules" + pipeline = Pipeline( redis_client=redis_client, # type: ignore[arg-type] rule_engine=RuleEngine(), geoip=geoip, on_result=dispatcher.dispatch, + inference_engine=inference_engine, + ensemble_weights={ + "ae": settings.ensemble_weight_ae, + "rf": settings.ensemble_weight_rf, + "if": settings.ensemble_weight_if, + }, raw_queue_size=settings.raw_queue_size, parsed_queue_size=settings.parsed_queue_size, feature_queue_size=settings.feature_queue_size, @@ -100,6 +114,34 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: logger.info("AngelusVigil shut down cleanly") +def _load_inference_engine() -> InferenceEngine | None: + """ + Attempt to load the ONNX inference engine from the + configured model directory, returning None if ML + dependencies are missing or no models are found + """ + try: + from app.core.detection.inference import ( + InferenceEngine, ) + except ImportError: + logger.info("onnxruntime not installed — running in rules-only mode") + return None + + engine = InferenceEngine(model_dir=settings.model_dir) + if engine.is_loaded: + logger.info( + "ML models loaded from %s", + settings.model_dir, + ) + return engine + + logger.info( + "No ML models found in %s — running in rules-only mode", + settings.model_dir, + ) + return None + + def create_app() -> FastAPI: """ Build and configure the AngelusVigil FastAPI application. @@ -114,12 +156,14 @@ def create_app() -> FastAPI: app.state.pipeline_running = False from app.api.health import router as health_router + from app.api.ingest import router as ingest_router from app.api.models_api import router as models_router from app.api.stats import router as stats_router from app.api.threats import router as threats_router from app.api.websocket import router as ws_router app.include_router(health_router) + app.include_router(ingest_router) app.include_router(threats_router) app.include_router(stats_router) app.include_router(models_router) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/models/model_metadata.py b/PROJECTS/advanced/ai-threat-detection/backend/app/models/model_metadata.py index 662c4045..6311239a 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/models/model_metadata.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/models/model_metadata.py @@ -15,14 +15,11 @@ class ModelMetadata(TimestampedModel, table=True): """ __tablename__ = "model_metadata" - __table_args__ = ( - Index( - "idx_model_metadata_active", - "model_type", - unique=True, - postgresql_where=text("is_active = TRUE"), - ), - ) + __table_args__ = (Index( + "idx_model_metadata_active", + "model_type", + postgresql_where=text("is_active = TRUE"), + ), ) model_type: str = Field(max_length=30) version: str = Field(max_length=64) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/models/threat_event.py b/PROJECTS/advanced/ai-threat-detection/backend/app/models/threat_event.py index b3e12cd6..f273ec28 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/models/threat_event.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/models/threat_event.py @@ -42,15 +42,16 @@ class ThreatEvent(TimestampedModel, table=True): user_agent: str threat_score: float = Field(sa_column=Column(Float, nullable=False)) severity: str = Field(max_length=6) - component_scores: dict[str, float] = Field(sa_column=Column(JSON, nullable=False)) + component_scores: dict[str, float] = Field( + sa_column=Column(JSON, nullable=False)) geo_country: str | None = Field(default=None, max_length=2) geo_city: str | None = Field(default=None, max_length=255) geo_lat: float | None = Field(default=None) geo_lon: float | None = Field(default=None) feature_vector: list[float] = Field(sa_column=Column(JSON, nullable=False)) - matched_rules: list[str] | None = Field( - default=None, sa_column=Column(JSON, nullable=True) - ) + matched_rules: list[str] | None = Field(default=None, + sa_column=Column(JSON, + nullable=True)) model_version: str | None = Field(default=None, max_length=64) reviewed: bool = Field(default=False) review_label: str | None = Field(default=None, max_length=20) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/stats.py b/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/stats.py index d8475dfd..5c74c46d 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/stats.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/stats.py @@ -40,7 +40,7 @@ class StatsResponse(BaseModel): """ time_range: str - total_requests: int + threats_stored: int threats_detected: int severity_breakdown: SeverityBreakdown top_source_ips: list[IPStatEntry] diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/services/stats_service.py b/PROJECTS/advanced/ai-threat-detection/backend/app/services/stats_service.py index 7d7404e2..25c7d4e7 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/services/stats_service.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/services/stats_service.py @@ -35,48 +35,49 @@ async def get_stats( delta = _RANGE_MAP.get(time_range, timedelta(hours=24)) cutoff = datetime.now(UTC) - delta - base = select(ThreatEvent).where(ThreatEvent.created_at >= cutoff) # type: ignore[arg-type] + base = select(ThreatEvent).where(ThreatEvent.created_at + >= cutoff) # type: ignore[arg-type] total_q = select(func.count()).select_from(base.subquery()) total = (await session.execute(total_q)).scalar_one() sev_q = ( - select(ThreatEvent.severity, func.count()) # type: ignore[call-overload] - .where(ThreatEvent.created_at >= cutoff) - .group_by(ThreatEvent.severity) - ) + select(ThreatEvent.severity, + func.count()) # type: ignore[call-overload] + .where(ThreatEvent.created_at >= cutoff).group_by( + ThreatEvent.severity)) sev_rows = (await session.execute(sev_q)).all() sev_map = {row[0]: row[1] for row in sev_rows} threats_detected = sev_map.get("HIGH", 0) + sev_map.get("MEDIUM", 0) ip_q = ( - select(ThreatEvent.source_ip, func.count().label("cnt")) # type: ignore[call-overload] - .where(ThreatEvent.created_at >= cutoff) - .group_by(ThreatEvent.source_ip) - .order_by(func.count().desc()) - .limit(10) - ) + select(ThreatEvent.source_ip, + func.count().label("cnt")) # type: ignore[call-overload] + .where(ThreatEvent.created_at >= cutoff).group_by( + ThreatEvent.source_ip).order_by(func.count().desc()).limit(10)) ip_rows = (await session.execute(ip_q)).all() path_q = ( - select(ThreatEvent.request_path, func.count().label("cnt")) # type: ignore[call-overload] - .where(ThreatEvent.created_at >= cutoff) - .group_by(ThreatEvent.request_path) - .order_by(func.count().desc()) - .limit(10) - ) + select(ThreatEvent.request_path, + func.count().label("cnt")) # type: ignore[call-overload] + .where(ThreatEvent.created_at >= cutoff).group_by( + ThreatEvent.request_path).order_by(func.count().desc()).limit(10)) path_rows = (await session.execute(path_q)).all() return StatsResponse( time_range=time_range, - total_requests=total, + threats_stored=total, threats_detected=threats_detected, severity_breakdown=SeverityBreakdown( high=sev_map.get("HIGH", 0), medium=sev_map.get("MEDIUM", 0), low=sev_map.get("LOW", 0), ), - top_source_ips=[IPStatEntry(source_ip=row[0], count=row[1]) for row in ip_rows], - top_attacked_paths=[PathStatEntry(path=row[0], count=row[1]) for row in path_rows], + top_source_ips=[ + IPStatEntry(source_ip=row[0], count=row[1]) for row in ip_rows + ], + top_attacked_paths=[ + PathStatEntry(path=row[0], count=row[1]) for row in path_rows + ], ) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py b/PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py index 503d19ee..d7971dd1 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py @@ -9,6 +9,7 @@ from datetime import datetime from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession +from app.core.detection.ensemble import classify_severity from app.core.ingestion.pipeline import ScoredRequest from app.models.threat_event import ThreatEvent from app.schemas.threats import ( @@ -63,19 +64,28 @@ async def get_threats( count_query = select(func.count()).select_from(ThreatEvent) if severity: - query = query.where(ThreatEvent.severity == severity.upper()) # type: ignore[arg-type] - count_query = count_query.where(ThreatEvent.severity == severity.upper()) # type: ignore[arg-type] + query = query.where( + ThreatEvent.severity == severity.upper()) # type: ignore[arg-type] + count_query = count_query.where( + ThreatEvent.severity == severity.upper()) # type: ignore[arg-type] if source_ip: - query = query.where(ThreatEvent.source_ip == source_ip) # type: ignore[arg-type] - count_query = count_query.where(ThreatEvent.source_ip == source_ip) # type: ignore[arg-type] + query = query.where( + ThreatEvent.source_ip == source_ip) # type: ignore[arg-type] + count_query = count_query.where( + ThreatEvent.source_ip == source_ip) # type: ignore[arg-type] if since: - query = query.where(ThreatEvent.created_at >= since) # type: ignore[arg-type] - count_query = count_query.where(ThreatEvent.created_at >= since) # type: ignore[arg-type] + query = query.where(ThreatEvent.created_at + >= since) # type: ignore[arg-type] + count_query = count_query.where(ThreatEvent.created_at + >= since) # type: ignore[arg-type] if until: - query = query.where(ThreatEvent.created_at <= until) # type: ignore[arg-type] - count_query = count_query.where(ThreatEvent.created_at <= until) # type: ignore[arg-type] + query = query.where(ThreatEvent.created_at + <= until) # type: ignore[arg-type] + count_query = count_query.where(ThreatEvent.created_at + <= until) # type: ignore[arg-type] - query = query.order_by(ThreatEvent.created_at.desc()) # type: ignore[attr-defined] + query = query.order_by( + ThreatEvent.created_at.desc()) # type: ignore[attr-defined] query = query.offset(offset).limit(limit) total = (await session.execute(count_query)).scalar_one() @@ -116,16 +126,16 @@ async def create_threat_event( status_code=scored.entry.status_code, response_size=scored.entry.response_size, user_agent=scored.entry.user_agent, - threat_score=scored.rule_result.threat_score, - severity=scored.rule_result.severity, + threat_score=scored.final_score, + severity=classify_severity(scored.final_score), component_scores=scored.rule_result.component_scores, - geo_country=scored.geo.country if scored.geo else None, - geo_city=scored.geo.city if scored.geo else None, - geo_lat=scored.geo.lat if scored.geo else None, - geo_lon=scored.geo.lon if scored.geo else None, + geo_country=(scored.geo.country if scored.geo else None), + geo_city=(scored.geo.city if scored.geo else None), + geo_lat=(scored.geo.lat if scored.geo else None), + geo_lon=(scored.geo.lon if scored.geo else None), feature_vector=scored.feature_vector, - matched_rules=scored.rule_result.matched_rules or None, - model_version="rules-v1", + matched_rules=(scored.rule_result.matched_rules or None), + model_version=scored.detection_mode, ) session.add(event) await session.flush() diff --git a/PROJECTS/advanced/ai-threat-detection/backend/cli/main.py b/PROJECTS/advanced/ai-threat-detection/backend/cli/main.py index a3d44c00..ab47502d 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/cli/main.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/cli/main.py @@ -3,6 +3,10 @@ main.py """ +import asyncio +import dataclasses +from pathlib import Path + import typer app = typer.Typer( @@ -11,15 +15,62 @@ app = typer.Typer( no_args_is_help=True, ) +DEFAULT_MODEL_DIR = "data/models" +DEFAULT_EPOCHS = 100 +DEFAULT_BATCH_SIZE = 256 +DEFAULT_SYNTHETIC_NORMAL = 1000 +DEFAULT_SYNTHETIC_ATTACK = 500 +DEFAULT_EXPERIMENT_NAME = "angelusvigil-training" +DEFAULT_SERVER_URL = "http://localhost:8000" + + +async def _write_metadata( + model_dir: Path, + training_samples: int, + metrics: dict[str, object], + mlflow_run_id: str | None, + threshold: float | None, +) -> None: + """ + Persist training metadata to the database + """ + from app.config import settings + from ml.metadata import save_model_metadata + from sqlalchemy.ext.asyncio import ( + AsyncSession, + async_sessionmaker, + create_async_engine, + ) + + engine = create_async_engine(settings.database_url) + try: + factory = async_sessionmaker( + engine, + class_=AsyncSession, + expire_on_commit=False, + ) + async with factory() as session: + await save_model_metadata( + session, + model_dir=model_dir, + training_samples=training_samples, + metrics=metrics, + mlflow_run_id=mlflow_run_id, + threshold=threshold, + ) + finally: + await engine.dispose() + @app.command() def serve( host: str = typer.Option("0.0.0.0", help="Bind address"), port: int = typer.Option(8000, help="Bind port"), - reload: bool = typer.Option(False, help="Enable auto-reload for development"), + reload: bool = typer.Option(False, + help="Enable auto-reload for development"), ) -> None: """ - Start the AngelusVigil API server. + Start the AngelusVigil API server """ import uvicorn @@ -31,16 +82,212 @@ def serve( ) +@app.command() +def train( + csic_dir: Path = typer.Option( + None, + help="Path to CSIC 2010 dataset directory", + ), + synthetic_normal: int = typer.Option( + DEFAULT_SYNTHETIC_NORMAL, + help="Number of synthetic normal samples", + ), + synthetic_attack: int = typer.Option( + DEFAULT_SYNTHETIC_ATTACK, + help="Number of synthetic attack samples", + ), + output_dir: Path = typer.Option( + DEFAULT_MODEL_DIR, + help="Directory to save ONNX models", + ), + epochs: int = typer.Option( + DEFAULT_EPOCHS, + help="Autoencoder training epochs", + ), + batch_size: int = typer.Option( + DEFAULT_BATCH_SIZE, + help="Training batch size", + ), + experiment_name: str = typer.Option( + DEFAULT_EXPERIMENT_NAME, + help="MLflow experiment name", + ), +) -> None: + """ + Train all ML models and export to ONNX + """ + import numpy as np + + from ml.orchestrator import TrainingOrchestrator + + X_parts: list[np.ndarray] = [] + y_parts: list[np.ndarray] = [] + + if csic_dir is not None: + if not csic_dir.exists(): + typer.echo( + f"Error: CSIC directory not found" + f" at {csic_dir}", + err=True, + ) + raise typer.Exit(code=1) + + from ml.data_loader import load_csic_dataset, load_csic_normal + + normal_path = csic_dir / "normalTrafficTraining.txt" + normal_test_path = csic_dir / "normalTrafficTest.txt" + attack_path = csic_dir / "anomalousTrafficTest.txt" + typer.echo(f"Loading CSIC data from {csic_dir}") + X_csic, y_csic = load_csic_dataset( + normal_path, attack_path + ) + X_parts.append(X_csic) + y_parts.append(y_csic) + typer.echo( + f" CSIC: {len(X_csic)} samples" + ) + + if normal_test_path.exists(): + X_extra, y_extra = load_csic_normal(normal_test_path) + X_parts.append(X_extra) + y_parts.append(y_extra) + typer.echo( + f" CSIC normal test: {len(X_extra)} samples" + ) + + if synthetic_normal > 0 or synthetic_attack > 0: + from ml.synthetic import generate_mixed_dataset + + typer.echo( + f"Generating synthetic data:" + f" {synthetic_normal} normal," + f" {synthetic_attack} attack" + ) + X_syn, y_syn = generate_mixed_dataset( + synthetic_normal, synthetic_attack + ) + X_parts.append(X_syn) + y_parts.append(y_syn) + + if not X_parts: + typer.echo( + "Error: no data sources specified", + err=True, + ) + raise typer.Exit(code=1) + + X = np.vstack(X_parts) + y = np.concatenate(y_parts) + typer.echo( + f"Total: {len(X)} samples" + f" ({int(np.sum(y == 0))} normal," + f" {int(np.sum(y == 1))} attack)" + ) + + orch = TrainingOrchestrator( + output_dir=Path(output_dir), + experiment_name=experiment_name, + epochs=epochs, + batch_size=batch_size, + ) + result = orch.run(X, y) + + try: + metrics: dict[str, object] = ( + dataclasses.asdict(result.ensemble_metrics) + if result.ensemble_metrics else {} + ) + asyncio.run(_write_metadata( + Path(output_dir), + int(len(X)), + metrics, + result.mlflow_run_id, + result.ae_metrics.get("ae_threshold"), + )) + typer.echo(" Model metadata saved to database") + except Exception as exc: + typer.echo( + f" Warning: could not save metadata to DB: {exc}", + err=True, + ) + + typer.echo(f"Models exported to {output_dir}") + if result.ensemble_metrics is not None: + typer.echo( + f" Ensemble F1:" + f" {result.ensemble_metrics.f1:.4f}" + ) + typer.echo( + f" Ensemble PR-AUC:" + f" {result.ensemble_metrics.pr_auc:.4f}" + ) + typer.echo( + f" Passed gates: {result.passed_gates}" + ) + + if not result.passed_gates: + raise typer.Exit(code=1) + + +@app.command() +def replay( + log_file: Path = typer.Option(..., + help="Path to nginx access log file"), + url: str = typer.Option( + DEFAULT_SERVER_URL, + help="Running server URL to send logs to", + ), + batch_size: int = typer.Option(100, help="Lines per batch"), +) -> None: + """ + Replay historical log lines through the pipeline + """ + import httpx + + if not log_file.exists(): + typer.echo( + f"Error: log file not found at {log_file}", + err=True, + ) + raise typer.Exit(code=1) + + lines = log_file.read_text().strip().splitlines() + typer.echo(f"Replaying {len(lines)} lines to {url}") + + sent = 0 + with httpx.Client(timeout=30.0) as client: + for i in range(0, len(lines), batch_size): + batch = lines[i:i + batch_size] + response = client.post( + f"{url}/ingest/batch", + json={"lines": batch}, + ) + if response.status_code == 200: + sent += len(batch) + else: + typer.echo( + f" Batch {i} failed: {response.status_code}", + err=True, + ) + + typer.echo(f"Replayed {sent}/{len(lines)} lines") + + @app.command() def config() -> None: """ - Print the current configuration (secrets redacted). + Print the current configuration (secrets redacted) """ from app.config import settings safe_fields = {} for key, value in settings.model_dump().items(): - if any(secret in key for secret in ("key", "password", "secret", "token")): + if any(secret in key for secret in ( + "key", + "password", + "secret", + "token", + )): safe_fields[key] = "***REDACTED***" elif "url" in key and "@" in str(value): safe_fields[key] = _redact_url(str(value)) @@ -54,12 +301,12 @@ def config() -> None: @app.command() def health( url: str = typer.Option( - "http://localhost:8000", + DEFAULT_SERVER_URL, help="Base URL of the running server", ), ) -> None: """ - Ping the running server's /health endpoint. + Ping the running server's /health endpoint """ import httpx @@ -69,18 +316,24 @@ def health( data = response.json() typer.echo(f" status: {data.get('status', 'unknown')}") typer.echo(f" uptime: {data.get('uptime_seconds', 0):.0f}s") - typer.echo(f" pipeline: {'running' if data.get('pipeline_running') else 'stopped'}") + typer.echo( + f" pipeline: {'running' if data.get('pipeline_running') else 'stopped'}" + ) except httpx.ConnectError: typer.echo("Error: cannot connect to server", err=True) raise typer.Exit(code=1) from None except httpx.HTTPStatusError as exc: - typer.echo(f"Error: server returned {exc.response.status_code}", err=True) + typer.echo( + f"Error: server returned {exc.response.status_code}", + err=True, + ) raise typer.Exit(code=1) from None def _redact_url(url: str) -> str: """ - Replace the user:password portion of a database URL with ***:***. + Replace the user:password portion of a database + URL with ***:*** """ if "://" not in url or "@" not in url: return url diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/autoencoder.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/autoencoder.py index 9be7ba4e..586468d3 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/ml/autoencoder.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/autoencoder.py @@ -45,20 +45,19 @@ class ThreatAutoencoder(nn.Module): nn.LeakyReLU(0.2), nn.Dropout(0.2), nn.Linear(24, input_dim), - nn.Sigmoid(), ) def encode(self, x: Tensor) -> Tensor: """ Compress input through the encoder to the 6-dim bottleneck. """ - return self.encoder(x) + return self.encoder(x) # type: ignore[no-any-return] def decode(self, z: Tensor) -> Tensor: """ Reconstruct input from the bottleneck representation. """ - return self.decoder(z) + return self.decoder(z) # type: ignore[no-any-return] def forward(self, x: Tensor) -> Tensor: """ @@ -71,4 +70,4 @@ class ThreatAutoencoder(nn.Module): Per-sample mean squared error between input and reconstruction. """ reconstructed = self.forward(x) - return torch.mean((x - reconstructed) ** 2, dim=1) + return torch.mean((x - reconstructed)**2, dim=1) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/data_loader.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/data_loader.py new file mode 100644 index 00000000..4e9454ba --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/data_loader.py @@ -0,0 +1,228 @@ +""" +©AngelaMos | 2026 +data_loader.py +""" + +import logging +import re +from dataclasses import dataclass +from datetime import datetime, UTC +from pathlib import Path + +import numpy as np + +from app.core.features.encoder import encode_for_inference +from app.core.features.extractor import extract_request_features +from app.core.features.mappings import WINDOWED_FEATURE_NAMES +from app.core.ingestion.parsers import ParsedLogEntry + +logger = logging.getLogger(__name__) + +_REQUEST_LINE_RE = re.compile( + r"^(GET|POST|PUT|DELETE|HEAD|OPTIONS|PATCH|TRACE)" + r"\s+(\S+)\s+(HTTP/\d\.\d)\s*$") + +_DEFAULT_IP = "192.168.1.100" + +_DEFAULT_UA = ("Mozilla/5.0 (compatible; Konqueror/3.5; Linux)" + " KHTML/3.5.8 (like Gecko)") + + +@dataclass +class CSICRequest: + """ + Single HTTP request parsed from CSIC 2010 dataset format + """ + + method: str + path: str + query_string: str + protocol: str + headers: dict[str, str] + body: str + label: int + + +def parse_csic_file( + path: Path, + label: int, +) -> list[CSICRequest]: + """ + Parse a CSIC 2010 dataset file into a list of CSICRequest objects + """ + text = path.read_text(encoding="utf-8", errors="replace") + lines = text.splitlines() + + blocks: list[list[str]] = [] + current: list[str] = [] + + for line in lines: + match = _REQUEST_LINE_RE.match(line) + if match and current: + blocks.append(current) + current = [line] + elif match: + current = [line] + elif current: + current.append(line) + + if current: + blocks.append(current) + + results: list[CSICRequest] = [] + for block in blocks: + req = _parse_request_block(block, label) + if req is not None: + results.append(req) + + logger.info( + "Parsed %d requests from %s (label=%d)", + len(results), + path.name, + label, + ) + return results + + +def _parse_request_block( + lines: list[str], + label: int, +) -> CSICRequest | None: + """ + Parse a single request block into a CSICRequest + """ + if not lines: + return None + + match = _REQUEST_LINE_RE.match(lines[0]) + if not match: + return None + + method = match.group(1) + full_uri = match.group(2) + protocol = match.group(3) + + if "?" in full_uri: + path, query_string = full_uri.split("?", 1) + else: + path = full_uri + query_string = "" + + headers: dict[str, str] = {} + body_start = len(lines) + + for i, line in enumerate(lines[1:], 1): + if not line.strip(): + body_start = i + 1 + break + if ": " in line: + key, value = line.split(": ", 1) + headers[key] = value + + body_lines = [ln for ln in lines[body_start:] if ln.strip()] + body = "\n".join(body_lines) + + return CSICRequest( + method=method, + path=path, + query_string=query_string, + protocol=protocol, + headers=headers, + body=body, + label=label, + ) + + +def csic_to_parsed_entry(req: CSICRequest) -> ParsedLogEntry: + """ + Convert a CSICRequest to a ParsedLogEntry with synthesized defaults + for fields not present in the CSIC dataset + """ + ua = req.headers.get("User-Agent", _DEFAULT_UA) + + query = req.query_string + if req.body: + query = (f"{query}&{req.body}" if query else req.body) + + return ParsedLogEntry( + ip=_DEFAULT_IP, + timestamp=datetime.now(UTC), + method=req.method, + path=req.path, + query_string=query, + status_code=200, + response_size=0, + referer="", + user_agent=ua, + raw_line="", + ) + + +def load_csic_dataset( + normal_path: Path, + attack_path: Path, +) -> tuple[np.ndarray, np.ndarray]: + """ + Load CSIC 2010 normal and attack files, extract features, + and return (X, y) arrays ready for model training + """ + normal_reqs = parse_csic_file(normal_path, label=0) + attack_reqs = parse_csic_file(attack_path, label=1) + + all_reqs = normal_reqs + attack_reqs + + vectors: list[list[float]] = [] + labels: list[int] = [] + + for req in all_reqs: + entry = csic_to_parsed_entry(req) + features = extract_request_features(entry) + + for name in WINDOWED_FEATURE_NAMES: + features[name] = 0.0 + + vector = encode_for_inference(features) + vectors.append(vector) + labels.append(req.label) + + X = np.array(vectors, dtype=np.float32) + y = np.array(labels, dtype=np.int32) + + logger.info( + "Dataset loaded: X=%s, y=%s (normal=%d, attack=%d)", + X.shape, + y.shape, + np.sum(y == 0), + np.sum(y == 1), + ) + + return X, y + + +def load_csic_normal( + path: Path, +) -> tuple[np.ndarray, np.ndarray]: + """ + Load a CSIC 2010 normal traffic file and return (X, y) arrays + with all labels set to 0 + """ + reqs = parse_csic_file(path, label=0) + + vectors: list[list[float]] = [] + for req in reqs: + entry = csic_to_parsed_entry(req) + features = extract_request_features(entry) + for name in WINDOWED_FEATURE_NAMES: + features[name] = 0.0 + vectors.append(encode_for_inference(features)) + + X = np.array(vectors, dtype=np.float32) + y = np.zeros(len(vectors), dtype=np.int32) + + logger.info( + "Loaded %d normal samples from %s", + len(vectors), + path.name, + ) + + return X, y diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/download_csic.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/download_csic.py new file mode 100644 index 00000000..75b1cb31 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/download_csic.py @@ -0,0 +1,108 @@ +""" +©AngelaMos | 2026 +download_csic.py +""" + +import hashlib +import logging +import sys +from pathlib import Path + +import httpx + +logger = logging.getLogger(__name__) + +DATASET_DIR = Path("data/datasets/csic2010") + +BASE_URL = ("https://gitlab.fing.edu.uy" + "/gsi/web-application-attacks-datasets" + "/-/raw/master/csic_2010") + +FILES = [ + "normalTrafficTraining.txt", + "normalTrafficTest.txt", + "anomalousTrafficTest.txt", +] + +MIN_FILE_BYTES = 1_000_000 + + +def _compute_sha256(path: Path) -> str: + """ + Compute SHA-256 hash of a file + """ + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(65536), b""): + h.update(chunk) + return h.hexdigest() + + +def download_csic(output_dir: Path = DATASET_DIR, ) -> None: + """ + Download CSIC 2010 dataset files + """ + output_dir.mkdir(parents=True, exist_ok=True) + + for filename in FILES: + dest = output_dir / filename + + if dest.exists() and dest.stat().st_size > MIN_FILE_BYTES: + logger.info("Skipping %s (already exists)", filename) + continue + + url = f"{BASE_URL}/{filename}" + logger.info("Downloading %s", url) + print(f"Downloading {filename}...") + + try: + with httpx.stream( + "GET", + url, + follow_redirects=True, + ) as response: + response.raise_for_status() + total = int( + response.headers.get("content-length", 0) + ) + downloaded = 0 + with open(dest, "wb") as f: + for chunk in response.iter_bytes( + chunk_size=65536 + ): + f.write(chunk) + downloaded += len(chunk) + if total > 0: + pct = min( + downloaded * 100 / total, 100 + ) + sys.stdout.write(f"\r {pct:.0f}%") + else: + mb = downloaded / 1_048_576 + sys.stdout.write(f"\r {mb:.1f} MB") + sys.stdout.flush() + print() + except Exception as exc: + logger.error( + "Failed to download %s: %s", + filename, + exc, + ) + print(f"\nError downloading {filename}: {exc}") + continue + + size = dest.stat().st_size + sha = _compute_sha256(dest) + print(f" Saved: {dest}" + f" ({size:,} bytes, sha256={sha[:12]})") + + if size < MIN_FILE_BYTES: + print(f" WARNING: {filename} is suspiciously" + f" small ({size:,} bytes)") + + print(f"\nDataset directory: {output_dir.resolve()}") + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + download_csic() diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/experiment.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/experiment.py new file mode 100644 index 00000000..c1758ad2 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/experiment.py @@ -0,0 +1,104 @@ +""" +©AngelaMos | 2026 +experiment.py +""" + +import platform +import subprocess +import sys +from pathlib import Path +from types import TracebackType + +import mlflow + + +class VigilExperiment: + """ + Context manager wrapping MLflow experiment runs + with automatic system metadata logging + """ + + def __init__(self, experiment_name: str) -> None: + self._experiment_name = experiment_name + self._run: mlflow.ActiveRun | None = None + self._run_id: str | None = None + + @property + def run_id(self) -> str | None: + """ + The MLflow run ID, set after entering context + """ + return self._run_id + + def __enter__(self) -> VigilExperiment: + mlflow.set_experiment(self._experiment_name) + self._run = mlflow.start_run() + self._run_id = self._run.info.run_id + self._log_system_metadata() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + if exc_type is not None: + mlflow.set_tag("status", "failed") + mlflow.set_tag("error", str(exc_val)[:500]) + else: + mlflow.set_tag("status", "completed") + mlflow.end_run() + self._run = None + + def log_params(self, params: dict[str, object]) -> None: + """ + Log a dictionary of parameters to the active run + """ + mlflow.log_params(params) + + def log_metrics( + self, + metrics: dict[str, float], + step: int | None = None, + ) -> None: + """ + Log a dictionary of metrics to the active run + """ + mlflow.log_metrics(metrics, step=step) + + def log_artifact(self, path: Path | str) -> None: + """ + Log a local file as an artifact + """ + mlflow.log_artifact(str(path)) + + def _log_system_metadata(self) -> None: + """ + Record Python version and git commit hash + """ + mlflow.set_tag("python_version", sys.version.split()[0]) + mlflow.set_tag("platform", platform.system()) + + git_hash = _get_git_hash() + if git_hash is not None: + mlflow.set_tag("git_commit", git_hash) + + +def _get_git_hash() -> str | None: + """ + Return the short git commit hash or None + """ + try: + result = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + timeout=5, + check=False, + ) + if result.returncode == 0: + return result.stdout.strip() + except FileNotFoundError: + pass + return None diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/export_onnx.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/export_onnx.py new file mode 100644 index 00000000..593dba8f --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/export_onnx.py @@ -0,0 +1,90 @@ +""" +©AngelaMos | 2026 +export_onnx.py +""" + +from pathlib import Path + +import torch +from skl2onnx import convert_sklearn +from skl2onnx.common.data_types import FloatTensorType +from sklearn.base import BaseEstimator + +from ml.autoencoder import ThreatAutoencoder + +ONNX_OPSET = 17 +SKL_TARGET_OPSET = {"": 17, "ai.onnx.ml": 3} + + +def export_autoencoder( + model: ThreatAutoencoder, + path: Path | str, + opset: int = ONNX_OPSET, +) -> Path: + """ + Export a PyTorch autoencoder to ONNX with dynamic batch dimension + """ + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + + model.eval() + dummy = torch.randn(1, model.input_dim) + + batch_dim = torch.export.Dim("batch_size", min=1) + + torch.onnx.export( + model, + dummy, # type: ignore[arg-type] + str(path), + opset_version=opset, + export_params=True, + do_constant_folding=True, + input_names=["features"], + output_names=["reconstructed"], + dynamic_shapes={"x": { + 0: batch_dim + }}, + ) + return path + + +def export_random_forest( + model: BaseEstimator, + n_features: int, + path: Path | str, +) -> Path: + """ + Export a sklearn random forest (or calibrated wrapper) to ONNX + """ + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + + initial_type = [("features", FloatTensorType([None, n_features]))] + onnx_model = convert_sklearn( + model, + initial_types=initial_type, + target_opset=SKL_TARGET_OPSET, + ) + path.write_bytes(onnx_model.SerializeToString()) + return path + + +def export_isolation_forest( + model: BaseEstimator, + n_features: int, + path: Path | str, +) -> Path: + """ + Export a sklearn isolation forest to ONNX + """ + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + + initial_type = [("features", FloatTensorType([None, n_features]))] + onnx_model = convert_sklearn( + model, + initial_types=initial_type, + target_opset=SKL_TARGET_OPSET, + ) + path.write_bytes(onnx_model.SerializeToString()) + return path diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/metadata.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/metadata.py new file mode 100644 index 00000000..9d3c4e22 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/metadata.py @@ -0,0 +1,81 @@ +""" +©AngelaMos | 2026 +metadata.py +""" + +import hashlib +import logging +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.model_metadata import ModelMetadata + +logger = logging.getLogger(__name__) + +MODEL_TYPES: dict[str, str] = { + "ae.onnx": "autoencoder", + "rf.onnx": "random_forest", + "if.onnx": "isolation_forest", +} + +VERSION_HASH_LENGTH = 12 + + +def compute_model_version(artifact_path: Path) -> str: + """ + Compute a 12-char hex version string from the SHA-256 hash of a file + """ + sha = hashlib.sha256(artifact_path.read_bytes()) + return sha.hexdigest()[:VERSION_HASH_LENGTH] + + +async def save_model_metadata( + session: AsyncSession, + model_dir: Path, + training_samples: int, + metrics: dict[str, object], + mlflow_run_id: str | None = None, + threshold: float | None = None, +) -> list[ModelMetadata]: + """ + Persist metadata for all 3 model types, deactivating previous active versions + """ + rows: list[ModelMetadata] = [] + + for filename, model_type in MODEL_TYPES.items(): + artifact_path = model_dir / filename + version = compute_model_version(artifact_path) + + result = await session.execute( + select(ModelMetadata).where( + ModelMetadata.model_type == model_type, # type: ignore[arg-type] + ModelMetadata.is_active == True, # type: ignore[arg-type] # noqa: E712 + )) + for old in result.scalars().all(): + old.is_active = False + await session.flush() + + row = ModelMetadata( + model_type=model_type, + version=version, + training_samples=training_samples, + metrics=metrics, + artifact_path=str(artifact_path), + is_active=True, + mlflow_run_id=mlflow_run_id, + threshold=threshold, + ) + session.add(row) + rows.append(row) + + await session.commit() + + logger.info( + "Saved metadata for %d models (samples=%d)", + len(rows), + training_samples, + ) + + return rows diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/orchestrator.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/orchestrator.py new file mode 100644 index 00000000..729321d9 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/orchestrator.py @@ -0,0 +1,240 @@ +""" +©AngelaMos | 2026 +orchestrator.py +""" + +import json +import logging +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + +from ml.experiment import VigilExperiment +from ml.export_onnx import ( + export_autoencoder, + export_isolation_forest, + export_random_forest, +) +from ml.splitting import prepare_training_data +from ml.train_autoencoder import train_autoencoder +from ml.train_classifiers import ( + train_isolation_forest, + train_random_forest, +) +from ml.validation import ValidationResult, validate_ensemble + +logger = logging.getLogger(__name__) + +N_FEATURES = 35 + +AE_FILENAME = "ae.onnx" +RF_FILENAME = "rf.onnx" +IF_FILENAME = "if.onnx" +SCALER_FILENAME = "scaler.json" +THRESHOLD_FILENAME = "threshold.json" + + +@dataclass +class TrainingResult: + """ + Aggregated results from a full training pipeline run + """ + + ae_metrics: dict[str, float] + rf_metrics: dict[str, float] + if_metrics: dict[str, float] + ensemble_metrics: ValidationResult | None + passed_gates: bool + output_dir: Path + mlflow_run_id: str | None + + +class TrainingOrchestrator: + """ + End-to-end training pipeline that splits data, trains all 3 models, + exports to ONNX, validates the ensemble, and logs to MLflow + """ + + def __init__( + self, + output_dir: Path, + experiment_name: str = "angelusvigil-training", + epochs: int = 100, + batch_size: int = 256, + ) -> None: + self._output_dir = output_dir + self._experiment_name = experiment_name + self._epochs = epochs + self._batch_size = batch_size + + def run( + self, + X: np.ndarray, + y: np.ndarray, + ) -> TrainingResult: + """ + Execute the full training pipeline + """ + self._output_dir.mkdir(parents=True, exist_ok=True) + + split = prepare_training_data(X, y) + + logger.info( + "Split: train=%d val=%d test=%d normal_train=%d", + len(split.X_train), + len(split.X_val), + len(split.X_test), + len(split.X_normal_train), + ) + + with VigilExperiment(self._experiment_name) as experiment: + experiment.log_params({ + "epochs": self._epochs, + "batch_size": self._batch_size, + "n_samples": len(X), + "n_attack": int(np.sum(y == 1)), + "n_normal": int(np.sum(y == 0)), + "n_features": X.shape[1], + }) + + ae_result = self._train_ae(split.X_normal_train) + ae_metrics = { + "ae_threshold": ae_result["threshold"], + "ae_final_train_loss": ae_result["history"]["train_loss"][-1], + "ae_final_val_loss": ae_result["history"]["val_loss"][-1], + } + + rf_result = self._train_rf(split.X_train, split.y_train) + rf_metrics = rf_result["metrics"] + + if_result = self._train_if(split.X_normal_train) + if_metrics = if_result["metrics"] + + self._export_models(ae_result, rf_result, if_result) + + experiment.log_metrics(ae_metrics) + experiment.log_metrics({ + f"rf_{k}": v + for k, v in rf_metrics.items() + }) + + try: + ensemble = validate_ensemble( + self._output_dir, + split.X_test, + split.y_test, + ) + experiment.log_metrics({ + "ensemble_precision": ensemble.precision, + "ensemble_recall": ensemble.recall, + "ensemble_f1": ensemble.f1, + "ensemble_pr_auc": ensemble.pr_auc, + "ensemble_roc_auc": ensemble.roc_auc, + }) + passed = ensemble.passed_gates + except Exception as exc: + logger.exception("Ensemble validation failed") + print( + f" WARNING: validation raised" + f" {type(exc).__name__}: {exc}", + file=sys.stderr, + ) + ensemble = None + passed = False + + for name in ( + AE_FILENAME, + RF_FILENAME, + IF_FILENAME, + SCALER_FILENAME, + THRESHOLD_FILENAME, + ): + experiment.log_artifact(self._output_dir / name) + + run_id = experiment.run_id + + logger.info( + "Training complete: passed_gates=%s run_id=%s", + passed, + run_id, + ) + + return TrainingResult( + ae_metrics=ae_metrics, + rf_metrics=rf_metrics, + if_metrics=if_metrics, + ensemble_metrics=ensemble, + passed_gates=passed, + output_dir=self._output_dir, + mlflow_run_id=run_id, + ) + + def _train_ae(self, X_normal: np.ndarray) -> dict[str, Any]: + """ + Train the autoencoder on normal-only data + """ + logger.info( + "Training autoencoder (%d epochs, %d samples)", + self._epochs, + len(X_normal), + ) + return train_autoencoder( + X_normal, + epochs=self._epochs, + batch_size=self._batch_size, + ) + + def _train_rf(self, X: np.ndarray, y: np.ndarray) -> dict[str, Any]: + """ + Train the random forest classifier + """ + logger.info( + "Training random forest (%d samples)", + len(X), + ) + return train_random_forest(X, y) + + def _train_if(self, X_normal: np.ndarray) -> dict[str, Any]: + """ + Train the isolation forest on normal-only data + """ + logger.info( + "Training isolation forest (%d samples)", + len(X_normal), + ) + return train_isolation_forest(X_normal) + + def _export_models( + self, + ae_result: dict[str, Any], + rf_result: dict[str, Any], + if_result: dict[str, Any], + ) -> None: + """ + Export all 3 models to ONNX and save scaler/threshold + """ + export_autoencoder( + ae_result["model"], + self._output_dir / AE_FILENAME, + ) + ae_result["scaler"].save_json(self._output_dir / SCALER_FILENAME) + threshold_data = {"threshold": ae_result["threshold"]} + (self._output_dir / THRESHOLD_FILENAME).write_text( + json.dumps(threshold_data, indent=2)) + + export_random_forest( + rf_result["model"], + N_FEATURES, + self._output_dir / RF_FILENAME, + ) + + export_isolation_forest( + if_result["model"], + N_FEATURES, + self._output_dir / IF_FILENAME, + ) + + logger.info("Exported models to %s", self._output_dir) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/scaler.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/scaler.py index 5a4909b6..3619161c 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/ml/scaler.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/scaler.py @@ -46,7 +46,7 @@ class FeatureScaler: """ if not self._fitted or self._scaler is None: raise RuntimeError("Scaler has not been fitted") - return self._scaler.transform(X).astype(np.float32) + return self._scaler.transform(X).astype(np.float32) # type: ignore[no-any-return] def inverse_transform(self, X: np.ndarray) -> np.ndarray: """ @@ -54,7 +54,7 @@ class FeatureScaler: """ if not self._fitted or self._scaler is None: raise RuntimeError("Scaler has not been fitted") - return self._scaler.inverse_transform(X).astype(np.float32) + return self._scaler.inverse_transform(X).astype(np.float32) # type: ignore[no-any-return] def fit_transform(self, X: np.ndarray) -> np.ndarray: """ @@ -74,14 +74,14 @@ class FeatureScaler: "scale": self._scaler.scale_.tolist(), "n_features": int(self._scaler.n_features_in_), } - Path(path).write_text(json.dumps(data, indent=2)) + Path(path).write_text(json.dumps(data, indent=2), encoding="utf-8") @classmethod def load_json(cls, path: Path | str) -> FeatureScaler: """ Reconstruct a fitted scaler from a JSON file. """ - data = json.loads(Path(path).read_text()) + data = json.loads(Path(path).read_text(encoding="utf-8")) scaler = cls() scaler._scaler = RobustScaler() scaler._scaler.center_ = np.array(data["center"], dtype=np.float64) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/splitting.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/splitting.py new file mode 100644 index 00000000..51470681 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/splitting.py @@ -0,0 +1,84 @@ +""" +©AngelaMos | 2026 +splitting.py +""" + +from dataclasses import dataclass + +import numpy as np +from imblearn.over_sampling import SMOTE +from sklearn.model_selection import train_test_split + + +@dataclass +class TrainingSplit: + """ + Result of stratified splitting with SMOTE oversampling + """ + + X_train: np.ndarray + y_train: np.ndarray + X_val: np.ndarray + y_val: np.ndarray + X_test: np.ndarray + y_test: np.ndarray + X_normal_train: np.ndarray + + +def prepare_training_data( + X: np.ndarray, + y: np.ndarray, + train_ratio: float = 0.70, + val_ratio: float = 0.15, + smote_strategy: float = 0.3, + smote_k: int = 5, + random_state: int = 42, +) -> TrainingSplit: + """ + Split data into train/val/test with SMOTE on training set only + """ + n_classes = len(np.unique(y)) + if n_classes < 2: + raise ValueError("y must contain at least 2 classes") + + test_size = 1.0 - train_ratio + X_train, X_rem, y_train, y_rem = train_test_split( + X, + y, + test_size=test_size, + stratify=y, + random_state=random_state, + ) + + X_val, X_test, y_val, y_test = train_test_split( + X_rem, + y_rem, + test_size=0.5, + stratify=y_rem, + random_state=random_state, + ) + + X_normal_train = X_train[y_train == 0] + + class_counts = np.bincount(y_train) + minority_count = class_counts.min() + majority_count = class_counts.max() + current_ratio = minority_count / majority_count + + if (minority_count >= smote_k + 1 and current_ratio < smote_strategy): + sampler = SMOTE( + sampling_strategy=smote_strategy, + k_neighbors=smote_k, + random_state=random_state, + ) + X_train, y_train = sampler.fit_resample(X_train, y_train) + + return TrainingSplit( + X_train=X_train, + y_train=y_train, + X_val=X_val, + y_val=y_val, + X_test=X_test, + y_test=y_test, + X_normal_train=X_normal_train, + ) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/synthetic.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/synthetic.py new file mode 100644 index 00000000..c916c3b7 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/synthetic.py @@ -0,0 +1,436 @@ +""" +©AngelaMos | 2026 +synthetic.py +""" + +import logging +import random +from datetime import UTC, datetime, timedelta + +import numpy as np + +from app.core.features.encoder import encode_for_inference +from app.core.features.extractor import extract_request_features +from app.core.features.mappings import WINDOWED_FEATURE_NAMES +from app.core.ingestion.parsers import ParsedLogEntry + +logger = logging.getLogger(__name__) + +SQLI_PAYLOADS: list[str] = [ + "' OR 1=1--", + "' OR '1'='1", + "' UNION SELECT NULL,NULL--", + "' UNION SELECT username,password FROM users--", + "1; DROP TABLE users--", + "admin'--", + "' AND 1=1--", + "' AND SLEEP(5)--", + "' OR BENCHMARK(1000000,SHA1('test'))--", + "1' ORDER BY 1--", + "1' ORDER BY 10--", + "' UNION ALL SELECT @@version--", + "-1' UNION SELECT 1,CONCAT(user(),database())--", + "' OR 'x'='x", + "1; WAITFOR DELAY '0:0:5'--", + "' AND EXTRACTVALUE(1,CONCAT(0x7e,version()))--", + "' AND UPDATEXML(1,CONCAT(0x7e,version()),1)--", + "admin' AND '1'='1", + "' UNION SELECT LOAD_FILE('/etc/passwd')--", + "' INTO OUTFILE '/tmp/shell.php'--", + "1' AND 1=1 UNION SELECT 1,2,3--", + "' OR EXISTS(SELECT * FROM users)--", +] + +XSS_PAYLOADS: list[str] = [ + "", + "", + "", + "", + "", + "javascript:alert(1)", + "