diff --git a/.gitignore b/.gitignore index 5d14ee32..009c88cc 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ *.cache *.coverage .env - +.angelusvigil/ diff --git a/PROJECTS/advanced/ai-threat-detection/.dockerignore b/PROJECTS/advanced/ai-threat-detection/.dockerignore new file mode 100644 index 00000000..b9535755 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/.dockerignore @@ -0,0 +1,27 @@ +# ©AngelaMos | 2026 +# .dockerignore + +.git +.gitignore +.env +.env.* +!.env.example + +.angelusvigil/ +docs/ + +backend/.venv/ +backend/.pytest_cache/ +backend/.mypy_cache/ +backend/.ruff_cache/ +backend/__pycache__/ +backend/**/__pycache__/ +backend/htmlcov/ +backend/.coverage +backend/*.egg-info/ + +*.log +*.swp +*.swo +*~ +.DS_Store diff --git a/PROJECTS/advanced/ai-threat-detection/.gitignore b/PROJECTS/advanced/ai-threat-detection/.gitignore index b74fa9bf..2e26837e 100644 --- a/PROJECTS/advanced/ai-threat-detection/.gitignore +++ b/PROJECTS/advanced/ai-threat-detection/.gitignore @@ -66,6 +66,7 @@ docker-compose.override.yml data/models/ data/geoip/ data/sample-logs/*.log +!data/sample-logs/sample-access.log # MLflow mlruns/ diff --git a/PROJECTS/advanced/ai-threat-detection/README.md b/PROJECTS/advanced/ai-threat-detection/README.md index 633956e0..c7748236 100644 --- a/PROJECTS/advanced/ai-threat-detection/README.md +++ b/PROJECTS/advanced/ai-threat-detection/README.md @@ -1,11 +1,20 @@ # AngelusVigil -IN PROGRESS +> **Status: IN PROGRESS** — Phase 1 complete, Phase 2 (ML models) next AI-powered threat detection engine that analyzes web server access logs using machine learning to classify HTTP traffic as benign or malicious in real-time. Deploys as a Docker sidecar alongside any nginx-based infrastructure. Zero code changes to the monitored application. +## Progress + +| Phase | Description | Status | +|-------|-------------|--------| +| Phase 1 | Core pipeline, rule-based detection, API, Docker | Complete | +| Phase 2 | ML ensemble (autoencoder + RF + IF), ONNX inference | Next | +| Phase 3 | Production hardening, monitoring, retraining | Planned | +| Phase 4 | Dashboard, active learning, explainability | Planned | + ## Tech Stack | Layer | Technology | @@ -20,8 +29,8 @@ Deploys as a Docker sidecar alongside any nginx-based infrastructure. Zero code ## Quick Start ```bash -just setup -just dev-up +docker compose -f dev.compose.yml up -d +curl http://localhost:36969/health ``` ## Architecture @@ -32,6 +41,8 @@ just dev-up - **MEDIUM** (0.5-0.7): Store + monitor - **LOW** (<0.5): Log only +Currently running rule-based detection (ModSecurity CRS patterns) as cold-start fallback until ML models are trained in Phase 2. + See `learn/` for detailed documentation. ## License diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/deps.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/deps.py new file mode 100644 index 00000000..adee7228 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/deps.py @@ -0,0 +1,20 @@ +""" +©AngelaMos | 2026 +deps.py +""" + +from collections.abc import AsyncIterator + +from fastapi import Request +from sqlalchemy.ext.asyncio import AsyncSession + + +async def get_session(request: Request) -> AsyncIterator[AsyncSession]: + """ + Yield an async database session from + the application's session factory + """ + factory = request.app.state.session_factory + async with factory() as session: + yield session + await session.commit() diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/health.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/health.py index a92602e5..d333162a 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/api/health.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/health.py @@ -6,6 +6,7 @@ health.py import time from fastapi import APIRouter, Request, Response +from sqlalchemy import text from app.core.redis_manager import redis_manager @@ -13,7 +14,7 @@ router = APIRouter() @router.get("/health") -async def health(request: Request) -> dict: +async def health(request: Request) -> dict[str, object]: """ Liveness probe — returns 200 if the process is alive. """ @@ -26,7 +27,7 @@ async def health(request: Request) -> dict: @router.get("/ready") -async def ready(request: Request, response: Response) -> dict: +async def ready(request: Request, response: Response) -> dict[str, object]: """ Readiness probe — checks all service dependencies. """ @@ -66,7 +67,7 @@ async def _check_database(request: Request) -> bool: return False try: async with engine.connect() as conn: - await conn.execute("SELECT 1") + await conn.execute(text("SELECT 1")) return True except Exception: return False diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py new file mode 100644 index 00000000..af634898 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/models_api.py @@ -0,0 +1,34 @@ +""" +©AngelaMos | 2026 +models_api.py +""" + +import uuid + +from fastapi import APIRouter + +router = APIRouter(prefix="/models", tags=["models"]) + + +@router.get("/status") +async def model_status() -> dict[str, object]: + """ + Return the status of active ML models. + """ + return { + "active_models": [], + "detection_mode": "rules-only", + "note": "ML models available after Phase 2 training", + } + + +@router.post("/retrain", status_code=202) +async def retrain() -> dict[str, object]: + """ + Trigger an async model retraining job. + """ + return { + "status": "accepted", + "job_id": uuid.uuid4().hex, + "note": "Retraining not available in Phase 1", + } diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/stats.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/stats.py new file mode 100644 index 00000000..73453acf --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/stats.py @@ -0,0 +1,24 @@ +""" +©AngelaMos | 2026 +stats.py +""" + +from fastapi import APIRouter, Depends, Query +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_session +from app.schemas.stats import StatsResponse +from app.services import stats_service + +router = APIRouter(prefix="/stats", tags=["stats"]) + + +@router.get("", response_model=StatsResponse) +async def get_stats( + session: AsyncSession = Depends(get_session), + time_range: str = Query("24h", alias="range"), +) -> StatsResponse: + """ + Aggregate threat statistics for a given time window. + """ + return await stats_service.get_stats(session, time_range) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/threats.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/threats.py new file mode 100644 index 00000000..7ee8d7ea --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/threats.py @@ -0,0 +1,54 @@ +""" +©AngelaMos | 2026 +threats.py +""" + +import uuid +from datetime import datetime + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_session +from app.schemas.threats import ThreatEventResponse, ThreatListResponse +from app.services import threat_service + +router = APIRouter(prefix="/threats", tags=["threats"]) + + +@router.get("", response_model=ThreatListResponse) +async def list_threats( + session: AsyncSession = Depends(get_session), + limit: int = Query(50, ge=1, le=100), + offset: int = Query(0, ge=0), + severity: str | None = Query(None), + source_ip: str | None = Query(None), + since: datetime | None = Query(None), + until: datetime | None = Query(None), +) -> ThreatListResponse: + """ + List threat events with optional filters and pagination. + """ + return await threat_service.get_threats( + session, + limit, + offset, + severity, + source_ip, + since, + until, + ) + + +@router.get("/{threat_id}", response_model=ThreatEventResponse) +async def get_threat( + threat_id: uuid.UUID, + session: AsyncSession = Depends(get_session), +) -> ThreatEventResponse: + """ + Fetch a single threat event by ID. + """ + result = await threat_service.get_threat_by_id(session, threat_id) + if result is None: + raise HTTPException(status_code=404, detail="Threat event not found") + return result diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/api/websocket.py b/PROJECTS/advanced/ai-threat-detection/backend/app/api/websocket.py new file mode 100644 index 00000000..60fe4c2c --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/api/websocket.py @@ -0,0 +1,63 @@ +""" +©AngelaMos | 2026 +websocket.py +""" + +import asyncio +import logging + +from fastapi import APIRouter, WebSocket, WebSocketDisconnect + +from app.core.alerts import ALERTS_CHANNEL +from app.core.redis_manager import redis_manager + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +@router.websocket("/ws/alerts") +async def ws_alerts(websocket: WebSocket) -> None: + """ + Stream real time threat alerts to connected WebSocket clients + via Redis pub/sub relay. + + Each client gets its own Redis subscriber so this works correctly + across multiple FastAPI workers. + """ + await websocket.accept() + + redis = redis_manager.client + if redis is None: + await websocket.close(code=1011, reason="Redis not available") + return + + pubsub = redis.pubsub() + await pubsub.subscribe(ALERTS_CHANNEL) + + async def _relay() -> None: + async for message in pubsub.listen(): + if message["type"] == "message": + await websocket.send_text(message["data"]) + + async def _receive() -> None: + try: + while True: + await websocket.receive() + except WebSocketDisconnect: + pass + + relay_task = asyncio.create_task(_relay()) + receive_task = asyncio.create_task(_receive()) + + try: + done, pending = await asyncio.wait( + [relay_task, receive_task], + return_when=asyncio.FIRST_COMPLETED, + ) + for task in pending: + task.cancel() + finally: + await pubsub.unsubscribe(ALERTS_CHANNEL) + await pubsub.aclose() # type: ignore[attr-defined] + logger.debug("WebSocket client disconnected") diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/config.py b/PROJECTS/advanced/ai-threat-detection/backend/app/config.py index 2995d902..8022e039 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/config.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/config.py @@ -25,9 +25,7 @@ class Settings(BaseSettings): api_key: str = "" log_level: str = "INFO" - database_url: str = ( - "postgresql+asyncpg://vigil:changeme@localhost:5432/angelusvigil" - ) + database_url: str = "postgresql+asyncpg://vigil:changeme@localhost:5432/angelusvigil" redis_url: str = "redis://localhost:6379" diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/__init__.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/__init__.py new file mode 100644 index 00000000..84399ce3 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/__init__.py @@ -0,0 +1,6 @@ +""" +©AngelaMos | 2026 +__init__.py +""" + +ALERTS_CHANNEL = "alerts" diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py new file mode 100644 index 00000000..c823842b --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/alerts/dispatcher.py @@ -0,0 +1,73 @@ +""" +©AngelaMos | 2026 +dispatcher.py +""" + +import logging + +import redis.asyncio as aioredis +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from app.core.alerts import ALERTS_CHANNEL +from app.core.ingestion.pipeline import ScoredRequest +from app.schemas.websocket import WebSocketAlert +from app.services.threat_service import create_threat_event + +logger = logging.getLogger(__name__) + + +class AlertDispatcher: + """ + Routes scored threat events to storage, pub/sub, and structured logging. + + MEDIUM+ severity events are persisted to PostgreSQL and published + to the Redis alerts channel for WebSocket relay. + All events are logged to stdout. + """ + + def __init__( + self, + redis_client: aioredis.Redis[str], + session_factory: async_sessionmaker[AsyncSession], + ) -> None: + self._redis = redis_client + self._session_factory = session_factory + + async def dispatch(self, scored: ScoredRequest) -> None: + """ + Handle a scored request from the pipeline's dispatch stage. + """ + logger.info( + "threat_event severity=%s score=%.2f ip=%s path=%s rules=%s", + scored.rule_result.severity, + scored.rule_result.threat_score, + scored.entry.ip, + scored.entry.path, + scored.rule_result.matched_rules, + ) + + if scored.rule_result.severity in ("HIGH", "MEDIUM"): + await self._store_event(scored) + await self._publish_alert(scored) + + async def _store_event(self, scored: ScoredRequest) -> None: + """ + Persist the scored request as a threat event in PostgreSQL. + """ + async with self._session_factory() as session: + await create_threat_event(session, scored) + await session.commit() + + async def _publish_alert(self, scored: ScoredRequest) -> None: + """ + Publish a real-time alert to the Redis pub/sub channel. + """ + alert = WebSocketAlert( + timestamp=scored.entry.timestamp, + source_ip=scored.entry.ip, + request_path=scored.entry.path, + threat_score=scored.rule_result.threat_score, + severity=scored.rule_result.severity, + component_scores=scored.rule_result.component_scores, + ) + await self._redis.publish(ALERTS_CHANNEL, alert.model_dump_json()) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/rules.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/rules.py index a1393f3f..92b41e8f 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/rules.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/detection/rules.py @@ -3,6 +3,7 @@ rules.py """ +import re from dataclasses import dataclass, field from typing import NamedTuple @@ -20,17 +21,19 @@ from app.core.ingestion.parsers import ParsedLogEntry class _PatternRule(NamedTuple): """ - A regex-based detection rule applied to the request URI. + A regex based detection rule applied to the request URI """ + name: str - pattern: object + pattern: re.Pattern[str] score: float class _ThresholdRule(NamedTuple): """ - A threshold-based detection rule applied to a windowed feature. + A threshold-based detection rule applied to a windowed feature """ + name: str feature_key: str threshold: float @@ -60,6 +63,7 @@ class RuleResult: """ Output of the rule-based detection engine for a single request. """ + threat_score: float severity: str matched_rules: list[str] = field(default_factory=list) @@ -81,7 +85,7 @@ class RuleEngine: """ Cold-start rule-based detection engine inspired by ModSecurity CRS. Scores requests using pattern matching, signature detection, - and behavioral thresholds from windowed features. + and behavioral thresholds from windowed features """ def score_request( @@ -109,10 +113,10 @@ class RuleEngine: if any(sig in ua_lower for sig in SCANNER_USER_AGENTS): matched.append(("SCANNER_UA", _SCANNER_UA_SCORE)) - for rule in _THRESHOLD_RULES: - value = features.get(rule.feature_key, 0) - if isinstance(value, (int, float)) and value > rule.threshold: - matched.append((rule.name, rule.score)) + for trule in _THRESHOLD_RULES: + value = features.get(trule.feature_key, 0) + if isinstance(value, (int, float)) and value > trule.threshold: + matched.append((trule.name, trule.score)) if not matched: return RuleResult(threat_score=0.0, severity="LOW") @@ -127,5 +131,5 @@ class RuleEngine: threat_score=threat_score, severity=_classify_severity(threat_score), matched_rules=[name for name, _ in matched], - component_scores={name: score for name, score in matched}, + component_scores=dict(matched), ) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/enrichment/geoip.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/enrichment/geoip.py index 6f67d164..cd2db230 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/enrichment/geoip.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/enrichment/geoip.py @@ -20,6 +20,7 @@ class GeoResult: """ Structured GeoIP lookup result. """ + country: str | None city: str | None lat: float | None diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/aggregator.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/aggregator.py index f7389980..5fd1bd23 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/aggregator.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/aggregator.py @@ -27,7 +27,7 @@ class WindowAggregator: Per-IP sliding window feature aggregator backed by Redis sorted sets. """ - def __init__(self, redis_client: aioredis.Redis) -> None: + def __init__(self, redis_client: aioredis.Redis[bytes]) -> None: self._redis = redis_client async def record_and_aggregate( @@ -66,17 +66,11 @@ class WindowAggregator: pipe.zadd(keys["requests"], {request_id: timestamp}) pipe.zadd(keys["paths"], {_hash_member(path): timestamp}) - pipe.zadd( - keys["statuses"], {f"{status_code}:{request_id}": timestamp} - ) + pipe.zadd(keys["statuses"], {f"{status_code}:{request_id}": timestamp}) pipe.zadd(keys["uas"], {_hash_member(user_agent): timestamp}) - pipe.zadd( - keys["sizes"], {f"{response_size}:{request_id}": timestamp} - ) + pipe.zadd(keys["sizes"], {f"{response_size}:{request_id}": timestamp}) pipe.zadd(keys["methods"], {f"{method}:{request_id}": timestamp}) - pipe.zadd( - keys["depths"], {f"{path_depth}:{request_id}": timestamp} - ) + pipe.zadd(keys["depths"], {f"{path_depth}:{request_id}": timestamp}) for key in keys.values(): pipe.zremrangebyscore(key, "-inf", trim_boundary) @@ -90,9 +84,7 @@ class WindowAggregator: pipe.zrangebyscore(keys["sizes"], w5m, "+inf") pipe.zrangebyscore(keys["methods"], w5m, "+inf") pipe.zrangebyscore(keys["depths"], w5m, "+inf") - pipe.zrangebyscore( - keys["requests"], w10m, "+inf", withscores=True - ) + pipe.zrangebyscore(keys["requests"], w10m, "+inf", withscores=True) for key in keys.values(): pipe.expire(key, KEY_TTL) @@ -135,9 +127,7 @@ def _error_rate(status_members: list[str]) -> float: """ if not status_members: return 0.0 - errors = sum( - 1 for m in status_members if int(m.split(":")[0]) >= 400 - ) + errors = sum(1 for m in status_members if int(m.split(":")[0]) >= 400) return errors / len(status_members) @@ -160,9 +150,7 @@ def _method_entropy(method_members: list[str]) -> float: methods = [m.split(":")[0] for m in method_members] counts = Counter(methods) total = len(methods) - return -sum( - (c / total) * math.log2(c / total) for c in counts.values() - ) + return -sum((c / total) * math.log2(c / total) for c in counts.values()) def _status_diversity(status_members: list[str]) -> float: @@ -195,10 +183,7 @@ def _inter_request_time_stats( if len(entries) < 2: return 0.0, 0.0 timestamps = sorted(score for _, score in entries) - deltas = [ - (timestamps[i + 1] - timestamps[i]) * 1000 - for i in range(len(timestamps) - 1) - ] + deltas = [(timestamps[i + 1] - timestamps[i]) * 1000 for i in range(len(timestamps) - 1)] mean = sum(deltas) / len(deltas) if len(deltas) < 2: return mean, 0.0 diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/encoder.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/encoder.py index d332c8e2..033b9157 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/encoder.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/encoder.py @@ -34,9 +34,9 @@ def encode_for_inference( if name in BOOLEAN_FEATURES: vector.append(1.0 if raw else 0.0) elif name in CATEGORICAL_ENCODERS: - vector.append(float(CATEGORICAL_ENCODERS[name].get(raw, 0))) + vector.append(float(CATEGORICAL_ENCODERS[name].get(str(raw), 0))) elif name == "country_code": - vector.append(_encode_country(raw)) + vector.append(_encode_country(str(raw))) else: vector.append(float(raw)) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/extractor.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/extractor.py index 71780bcd..25f493eb 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/extractor.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/extractor.py @@ -25,9 +25,7 @@ def _shannon_entropy(s: str) -> float: return 0.0 length = len(s) counts = Counter(s) - return -sum( - (c / length) * math.log2(c / length) for c in counts.values() - ) + return -sum((c / length) * math.log2(c / length) for c in counts.values()) def _is_private_ip(ip_str: str) -> bool: @@ -63,9 +61,7 @@ def extract_request_features( "path_entropy": _shannon_entropy(entry.path), "path_length": path_len, "query_string_length": len(entry.query_string), - "query_param_count": ( - len(entry.query_string.split("&")) if entry.query_string else 0 - ), + "query_param_count": (len(entry.query_string.split("&")) if entry.query_string else 0), "has_encoded_chars": bool(ENCODED_CHARS.search(full_uri)), "has_double_encoding": bool(DOUBLE_ENCODED.search(full_uri)), "status_code": entry.status_code, @@ -77,9 +73,7 @@ def extract_request_features( "ua_length": len(entry.user_agent), "ua_entropy": _shannon_entropy(entry.user_agent), "is_known_bot": any(sig in ua_lower for sig in BOT_USER_AGENTS), - "is_known_scanner": any( - sig in ua_lower for sig in SCANNER_USER_AGENTS - ), + "is_known_scanner": any(sig in ua_lower for sig in SCANNER_USER_AGENTS), "has_attack_pattern": bool(ATTACK_COMBINED.search(full_uri)), "special_char_ratio": non_alnum / path_len if path_len else 0.0, "file_extension": ext, diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/mappings.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/mappings.py index 5f91d496..829f5584 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/mappings.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/mappings.py @@ -93,12 +93,14 @@ CATEGORICAL_ENCODERS: dict[str, dict[str, int]] = { "file_extension": EXTENSION_MAP, } -BOOLEAN_FEATURES: frozenset[str] = frozenset({ - "has_encoded_chars", - "has_double_encoding", - "is_weekend", - "is_known_bot", - "is_known_scanner", - "has_attack_pattern", - "is_private_ip", -}) +BOOLEAN_FEATURES: frozenset[str] = frozenset( + { + "has_encoded_chars", + "has_double_encoding", + "is_weekend", + "is_known_bot", + "is_known_scanner", + "has_attack_pattern", + "is_private_ip", + } +) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/signatures.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/signatures.py index 2c6a5bb1..d1ce3ed0 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/signatures.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/features/signatures.py @@ -3,83 +3,87 @@ signatures.py """ -BOT_USER_AGENTS: frozenset[str] = frozenset({ - "googlebot", - "bingbot", - "slurp", - "duckduckbot", - "baiduspider", - "yandexbot", - "sogou", - "facebot", - "ia_archiver", - "applebot", - "petalbot", - "semrushbot", - "ahrefsbot", - "mj12bot", - "dotbot", - "rogerbot", - "linkedinbot", - "twitterbot", - "gptbot", - "claudebot", - "amazonbot", - "bytespider", - "ccbot", - "dataforseo", - "seznambot", - "megaindex", - "blexbot", - "exabot", - "archive.org_bot", - "mojeekbot", - "uptimerobot", - "deadlinkchecker", - "sitebulb", - "screaming frog", -}) +BOT_USER_AGENTS: frozenset[str] = frozenset( + { + "googlebot", + "bingbot", + "slurp", + "duckduckbot", + "baiduspider", + "yandexbot", + "sogou", + "facebot", + "ia_archiver", + "applebot", + "petalbot", + "semrushbot", + "ahrefsbot", + "mj12bot", + "dotbot", + "rogerbot", + "linkedinbot", + "twitterbot", + "gptbot", + "claudebot", + "amazonbot", + "bytespider", + "ccbot", + "dataforseo", + "seznambot", + "megaindex", + "blexbot", + "exabot", + "archive.org_bot", + "mojeekbot", + "uptimerobot", + "deadlinkchecker", + "sitebulb", + "screaming frog", + } +) -SCANNER_USER_AGENTS: frozenset[str] = frozenset({ - "nikto", - "sqlmap", - "nessus", - "openvas", - "acunetix", - "w3af", - "nmap", - "masscan", - "zgrab", - "gobuster", - "dirbuster", - "dirb", - "wfuzz", - "ffuf", - "nuclei", - "burp", - "zap", - "arachni", - "skipfish", - "wpscan", - "joomscan", - "whatweb", - "httprint", - "fierce", - "subfinder", - "amass", - "httpx", - "jaeles", - "xray", - "gau", - "hakrawler", - "katana", - "cariddi", - "gospider", - "feroxbuster", - "rustbuster", - "patator", - "hydra", - "medusa", - "metasploit", - "cobalt", -}) +SCANNER_USER_AGENTS: frozenset[str] = frozenset( + { + "nikto", + "sqlmap", + "nessus", + "openvas", + "acunetix", + "w3af", + "nmap", + "masscan", + "zgrab", + "gobuster", + "dirbuster", + "dirb", + "wfuzz", + "ffuf", + "nuclei", + "burp", + "zap", + "arachni", + "skipfish", + "wpscan", + "joomscan", + "whatweb", + "httprint", + "fierce", + "subfinder", + "amass", + "httpx", + "jaeles", + "xray", + "gau", + "hakrawler", + "katana", + "cariddi", + "gospider", + "feroxbuster", + "rustbuster", + "patator", + "hydra", + "medusa", + "metasploit", + "cobalt", + } +) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/parsers.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/parsers.py index b80a98cf..00ea360e 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/parsers.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/parsers.py @@ -72,9 +72,7 @@ def _parse_split(line: str) -> ParsedLogEntry | None: bracket_open = prefix.index("[") bracket_close = prefix.index("]") ip = prefix[:bracket_open].split()[0] - timestamp = datetime.strptime( - prefix[bracket_open + 1 : bracket_close], _TIMESTAMP_FMT - ) + timestamp = datetime.strptime(prefix[bracket_open + 1 : bracket_close], _TIMESTAMP_FMT) request_parts = request_line.split(" ", 2) method = request_parts[0] diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py index e5ecb76a..4ccf34de 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/pipeline.py @@ -26,6 +26,7 @@ class EnrichedRequest: """ A parsed log entry enriched with extracted features and GeoIP data. """ + entry: ParsedLogEntry features: dict[str, int | float | bool | str] feature_vector: list[float] @@ -37,6 +38,7 @@ class ScoredRequest: """ A fully scored request ready for dispatch. """ + entry: ParsedLogEntry features: dict[str, int | float | bool | str] feature_vector: list[float] @@ -56,7 +58,7 @@ class Pipeline: def __init__( self, - redis_client: aioredis.Redis, + redis_client: aioredis.Redis[bytes], rule_engine: RuleEngine, geoip: GeoIPService | None = None, on_result: Callable[[ScoredRequest], Awaitable[None]] | None = None, @@ -82,7 +84,7 @@ class Pipeline: self._rule_engine = rule_engine self._geoip = geoip self._on_result = on_result - self._tasks: list[asyncio.Task] = [] + self._tasks: list[asyncio.Task[None]] = [] async def start(self) -> None: """ @@ -147,7 +149,7 @@ class Pipeline: ip=entry.ip, request_id=uuid.uuid4().hex, path=entry.path, - path_depth=per_request["path_depth"], + path_depth=int(per_request["path_depth"]), method=entry.method, status_code=entry.status_code, user_agent=entry.user_agent, @@ -182,7 +184,8 @@ class Pipeline: break try: rule_result = self._rule_engine.score_request( - enriched.features, enriched.entry, + enriched.features, + enriched.entry, ) await self._alert_queue.put( ScoredRequest( diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py index 3f2c9370..3d0674fa 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/ingestion/tailer.py @@ -6,6 +6,7 @@ tailer.py import asyncio import logging import os +from io import TextIOWrapper from pathlib import Path from watchdog.events import ( @@ -28,14 +29,14 @@ class _LogHandler(FileSystemEventHandler): def __init__( self, target: str, - queue: asyncio.Queue[str], + queue: asyncio.Queue[str | None], loop: asyncio.AbstractEventLoop, ) -> None: super().__init__() self._target = target self._queue = queue self._loop = loop - self._file = None + self._file: TextIOWrapper | None = None self._inode: int | None = None self._open_target() @@ -44,7 +45,7 @@ class _LogHandler(FileSystemEventHandler): Open the target log file and seek to the end. """ try: - self._file = open(self._target, encoding="utf-8", errors="replace") + self._file = open(self._target, encoding="utf-8", errors="replace") # noqa: SIM115 self._file.seek(0, os.SEEK_END) self._inode = os.stat(self._target).st_ino logger.info("Tailing %s (inode %s)", self._target, self._inode) @@ -75,7 +76,7 @@ class _LogHandler(FileSystemEventHandler): self._file.close() try: - self._file = open(self._target, encoding="utf-8", errors="replace") + self._file = open(self._target, encoding="utf-8", errors="replace") # noqa: SIM115 self._inode = os.stat(self._target).st_ino logger.info("Rotated to new %s (inode %s)", self._target, self._inode) except FileNotFoundError: @@ -92,14 +93,14 @@ class _LogHandler(FileSystemEventHandler): except FileNotFoundError: return False - def on_modified(self, event: FileModifiedEvent) -> None: + def on_modified(self, event: FileModifiedEvent) -> None: # type: ignore[override] """ Handle new data appended to the log file. """ if not isinstance(event, FileModifiedEvent) or event.is_directory: return - if Path(event.src_path).resolve() != Path(self._target).resolve(): + if Path(str(event.src_path)).resolve() != Path(self._target).resolve(): return if self._inode_changed(): @@ -108,25 +109,25 @@ class _LogHandler(FileSystemEventHandler): self._read_new_lines() - def on_moved(self, event: FileMovedEvent) -> None: + def on_moved(self, event: FileMovedEvent) -> None: # type: ignore[override] """ Handle log rotation via rename (access.log -> access.log.1). """ if not isinstance(event, FileMovedEvent): return - if Path(event.src_path).resolve() == Path(self._target).resolve(): + if Path(str(event.src_path)).resolve() == Path(self._target).resolve(): logger.info("Log rotated: %s -> %s", event.src_path, event.dest_path) self._handle_rotation() - def on_created(self, event: FileCreatedEvent) -> None: + def on_created(self, event: FileCreatedEvent) -> None: # type: ignore[override] """ Handle log rotation where a new file is created at the target path. """ if not isinstance(event, FileCreatedEvent) or event.is_directory: return - if Path(event.src_path).resolve() == Path(self._target).resolve(): + if Path(str(event.src_path)).resolve() == Path(self._target).resolve(): logger.info("New log file created: %s", event.src_path) self._handle_rotation() @@ -148,7 +149,7 @@ class LogTailer: def __init__( self, log_path: str, - queue: asyncio.Queue[str], + queue: asyncio.Queue[str | None], loop: asyncio.AbstractEventLoop, ) -> None: self._log_path = log_path diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/core/redis_manager.py b/PROJECTS/advanced/ai-threat-detection/backend/app/core/redis_manager.py index f93bb006..163c1a10 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/core/redis_manager.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/core/redis_manager.py @@ -14,7 +14,7 @@ class RedisManager: """ def __init__(self) -> None: - self._client: aioredis.Redis | None = None + self._client: aioredis.Redis[str] | None = None async def connect(self) -> None: """ @@ -30,11 +30,11 @@ class RedisManager: Close the Redis connection and release resources. """ if self._client: - await self._client.aclose() + await self._client.aclose() # type: ignore[attr-defined] self._client = None @property - def client(self) -> aioredis.Redis | None: + def client(self) -> aioredis.Redis[str] | None: """ Return the active Redis client or None if not connected. """ diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py b/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py index 0edd0090..95694023 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/factory.py @@ -3,34 +3,106 @@ factory.py """ +import asyncio +import logging import time from collections.abc import AsyncIterator from contextlib import asynccontextmanager +from pathlib import Path from fastapi import FastAPI +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlmodel import SQLModel from app.config import settings +from app.core.alerts.dispatcher import AlertDispatcher +from app.core.detection.rules import RuleEngine +from app.core.enrichment.geoip import GeoIPService +from app.core.ingestion.pipeline import Pipeline +from app.core.ingestion.tailer import LogTailer from app.core.redis_manager import redis_manager +from app.models import model_metadata as _model_metadata_reg # noqa: F401 +from app.models import threat_event as _threat_event_reg # noqa: F401 + +logger = logging.getLogger(__name__) @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: """ - Manage application startup and shutdown lifecycle + Manage application startup and shutdown lifecycle. """ app.state.startup_time = time.monotonic() app.state.pipeline_running = False + engine = create_async_engine(settings.database_url) + app.state.db_engine = engine + app.state.session_factory = async_sessionmaker( + engine, + class_=AsyncSession, + expire_on_commit=False, + ) + + async with engine.begin() as conn: + await conn.run_sync(SQLModel.metadata.create_all) + logger.info("Database tables verified") + await redis_manager.connect() + geoip = GeoIPService(settings.geoip_db_path) + + redis_client = redis_manager.client + assert redis_client is not None + + dispatcher = AlertDispatcher( + redis_client=redis_client, + session_factory=app.state.session_factory, + ) + + pipeline = Pipeline( + redis_client=redis_client, # type: ignore[arg-type] + rule_engine=RuleEngine(), + geoip=geoip, + on_result=dispatcher.dispatch, + raw_queue_size=settings.raw_queue_size, + parsed_queue_size=settings.parsed_queue_size, + feature_queue_size=settings.feature_queue_size, + alert_queue_size=settings.alert_queue_size, + ) + await pipeline.start() + + tailer = None + log_dir = Path(settings.nginx_log_path).resolve().parent + if log_dir.is_dir(): + loop = asyncio.get_running_loop() + tailer = LogTailer(settings.nginx_log_path, pipeline.raw_queue, loop) + tailer.start() + else: + logger.warning("Log directory %s not found — tailer disabled", log_dir) + + app.state.pipeline = pipeline + app.state.tailer = tailer + app.state.geoip = geoip + app.state.pipeline_running = True + + logger.info("AngelusVigil started — pipeline active") + yield + app.state.pipeline_running = False + if tailer is not None: + tailer.stop() + await pipeline.stop() + geoip.close() await redis_manager.disconnect() + await engine.dispose() + + logger.info("AngelusVigil shut down cleanly") def create_app() -> FastAPI: """ - Build and configure the AngelusVigil FastAPI application + Build and configure the AngelusVigil FastAPI application. """ app = FastAPI( title=settings.app_name, @@ -42,8 +114,15 @@ def create_app() -> FastAPI: app.state.pipeline_running = False from app.api.health import router as health_router + from app.api.models_api import router as models_router + from app.api.stats import router as stats_router + from app.api.threats import router as threats_router + from app.api.websocket import router as ws_router app.include_router(health_router) + app.include_router(threats_router) + app.include_router(stats_router) + app.include_router(models_router) + app.include_router(ws_router) return app - diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/models/base.py b/PROJECTS/advanced/ai-threat-detection/backend/app/models/base.py index 8f559c3a..764cdc49 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/models/base.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/models/base.py @@ -16,7 +16,7 @@ class TimestampedModel(SQLModel): """ id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) - created_at: datetime = Field( + created_at: datetime = Field( # type: ignore[call-overload] sa_type=DateTime(timezone=True), sa_column_kwargs={"server_default": text("CURRENT_TIMESTAMP")}, nullable=False, diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/models/model_metadata.py b/PROJECTS/advanced/ai-threat-detection/backend/app/models/model_metadata.py index 867013e1..662c4045 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/models/model_metadata.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/models/model_metadata.py @@ -27,7 +27,7 @@ class ModelMetadata(TimestampedModel, table=True): model_type: str = Field(max_length=30) version: str = Field(max_length=64) training_samples: int - metrics: dict = Field(sa_column=Column(JSON, nullable=False)) + metrics: dict[str, object] = Field(sa_column=Column(JSON, nullable=False)) artifact_path: str is_active: bool = Field(default=False) mlflow_run_id: str | None = Field(default=None, max_length=64) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/models/threat_event.py b/PROJECTS/advanced/ai-threat-detection/backend/app/models/threat_event.py index a64c6cd7..b3e12cd6 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/app/models/threat_event.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/models/threat_event.py @@ -3,7 +3,14 @@ threat_event.py """ -from sqlalchemy import Column, Float, Index, JSON, SmallInteger, text +from sqlalchemy import ( + Column, + Float, + Index, + JSON, + SmallInteger, + text, +) from sqlmodel import Field from app.models.base import TimestampedModel @@ -30,25 +37,17 @@ class ThreatEvent(TimestampedModel, table=True): source_ip: str = Field(max_length=45) request_method: str = Field(max_length=10) request_path: str - status_code: int = Field( - sa_column=Column(SmallInteger, nullable=False) - ) + status_code: int = Field(sa_column=Column(SmallInteger, nullable=False)) response_size: int user_agent: str - threat_score: float = Field( - sa_column=Column(Float, nullable=False) - ) + threat_score: float = Field(sa_column=Column(Float, nullable=False)) severity: str = Field(max_length=6) - component_scores: dict[str, float] = Field( - sa_column=Column(JSON, nullable=False) - ) + component_scores: dict[str, float] = Field(sa_column=Column(JSON, nullable=False)) geo_country: str | None = Field(default=None, max_length=2) geo_city: str | None = Field(default=None, max_length=255) geo_lat: float | None = Field(default=None) geo_lon: float | None = Field(default=None) - feature_vector: list[float] = Field( - sa_column=Column(JSON, nullable=False) - ) + feature_vector: list[float] = Field(sa_column=Column(JSON, nullable=False)) matched_rules: list[str] | None = Field( default=None, sa_column=Column(JSON, nullable=True) ) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/__init__.py b/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/__init__.py new file mode 100644 index 00000000..e1add2a9 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/__init__.py @@ -0,0 +1,4 @@ +""" +©AngelaMos | 2026 +__init__.py +""" diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/stats.py b/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/stats.py new file mode 100644 index 00000000..d8475dfd --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/stats.py @@ -0,0 +1,47 @@ +""" +©AngelaMos | 2026 +stats.py +""" + +from pydantic import BaseModel + + +class SeverityBreakdown(BaseModel): + """ + Count of threats per severity tier. + """ + + high: int = 0 + medium: int = 0 + low: int = 0 + + +class IPStatEntry(BaseModel): + """ + Source IP with associated threat count. + """ + + source_ip: str + count: int + + +class PathStatEntry(BaseModel): + """ + Request path with associated threat count. + """ + + path: str + count: int + + +class StatsResponse(BaseModel): + """ + Aggregate threat statistics for a given time range. + """ + + time_range: str + total_requests: int + threats_detected: int + severity_breakdown: SeverityBreakdown + top_source_ips: list[IPStatEntry] + top_attacked_paths: list[PathStatEntry] diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/threats.py b/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/threats.py new file mode 100644 index 00000000..b2d42fe9 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/threats.py @@ -0,0 +1,57 @@ +""" +©AngelaMos | 2026 +threats.py +""" + +import uuid +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel + + +class GeoInfo(BaseModel): + """ + Geographic location data from GeoIP lookup. + """ + + country: str | None = None + city: str | None = None + lat: float | None = None + lon: float | None = None + + +class ThreatEventResponse(BaseModel): + """ + Single threat event returned by the API. + """ + + id: uuid.UUID + created_at: datetime + source_ip: str + request_method: str + request_path: str + status_code: int + response_size: int + user_agent: str + threat_score: float + severity: Literal["HIGH", "MEDIUM", "LOW"] + component_scores: dict[str, float] + geo: GeoInfo + matched_rules: list[str] | None = None + model_version: str | None = None + reviewed: bool = False + review_label: str | None = None + + model_config = {"from_attributes": True} + + +class ThreatListResponse(BaseModel): + """ + Paginated list of threat events. + """ + + total: int + limit: int + offset: int + items: list[ThreatEventResponse] diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/websocket.py b/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/websocket.py new file mode 100644 index 00000000..8c7a4efa --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/schemas/websocket.py @@ -0,0 +1,23 @@ +""" +©AngelaMos | 2026 +websocket.py +""" + +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel + + +class WebSocketAlert(BaseModel): + """ + Real-time threat alert broadcast over WebSocket. + """ + + event: Literal["threat"] = "threat" + timestamp: datetime + source_ip: str + request_path: str + threat_score: float + severity: str + component_scores: dict[str, float] diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/services/__init__.py b/PROJECTS/advanced/ai-threat-detection/backend/app/services/__init__.py new file mode 100644 index 00000000..e1add2a9 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/services/__init__.py @@ -0,0 +1,4 @@ +""" +©AngelaMos | 2026 +__init__.py +""" diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/services/stats_service.py b/PROJECTS/advanced/ai-threat-detection/backend/app/services/stats_service.py new file mode 100644 index 00000000..7d7404e2 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/services/stats_service.py @@ -0,0 +1,82 @@ +""" +©AngelaMos | 2026 +stats_service.py +""" + +from datetime import datetime, timedelta, UTC + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.threat_event import ThreatEvent +from app.schemas.stats import ( + IPStatEntry, + PathStatEntry, + SeverityBreakdown, + StatsResponse, +) + +_RANGE_MAP: dict[str, timedelta] = { + "1h": timedelta(hours=1), + "6h": timedelta(hours=6), + "24h": timedelta(hours=24), + "7d": timedelta(days=7), + "30d": timedelta(days=30), +} + + +async def get_stats( + session: AsyncSession, + time_range: str = "24h", +) -> StatsResponse: + """ + Compute aggregate threat statistics for a given time window. + """ + delta = _RANGE_MAP.get(time_range, timedelta(hours=24)) + cutoff = datetime.now(UTC) - delta + + base = select(ThreatEvent).where(ThreatEvent.created_at >= cutoff) # type: ignore[arg-type] + + total_q = select(func.count()).select_from(base.subquery()) + total = (await session.execute(total_q)).scalar_one() + + sev_q = ( + select(ThreatEvent.severity, func.count()) # type: ignore[call-overload] + .where(ThreatEvent.created_at >= cutoff) + .group_by(ThreatEvent.severity) + ) + sev_rows = (await session.execute(sev_q)).all() + sev_map = {row[0]: row[1] for row in sev_rows} + + threats_detected = sev_map.get("HIGH", 0) + sev_map.get("MEDIUM", 0) + + ip_q = ( + select(ThreatEvent.source_ip, func.count().label("cnt")) # type: ignore[call-overload] + .where(ThreatEvent.created_at >= cutoff) + .group_by(ThreatEvent.source_ip) + .order_by(func.count().desc()) + .limit(10) + ) + ip_rows = (await session.execute(ip_q)).all() + + path_q = ( + select(ThreatEvent.request_path, func.count().label("cnt")) # type: ignore[call-overload] + .where(ThreatEvent.created_at >= cutoff) + .group_by(ThreatEvent.request_path) + .order_by(func.count().desc()) + .limit(10) + ) + path_rows = (await session.execute(path_q)).all() + + return StatsResponse( + time_range=time_range, + total_requests=total, + threats_detected=threats_detected, + severity_breakdown=SeverityBreakdown( + high=sev_map.get("HIGH", 0), + medium=sev_map.get("MEDIUM", 0), + low=sev_map.get("LOW", 0), + ), + top_source_ips=[IPStatEntry(source_ip=row[0], count=row[1]) for row in ip_rows], + top_attacked_paths=[PathStatEntry(path=row[0], count=row[1]) for row in path_rows], + ) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py b/PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py new file mode 100644 index 00000000..503d19ee --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/app/services/threat_service.py @@ -0,0 +1,132 @@ +""" +©AngelaMos | 2026 +threat_service.py +""" + +import uuid +from datetime import datetime + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.ingestion.pipeline import ScoredRequest +from app.models.threat_event import ThreatEvent +from app.schemas.threats import ( + GeoInfo, + ThreatEventResponse, + ThreatListResponse, +) + + +def _to_response(event: ThreatEvent) -> ThreatEventResponse: + """ + Convert a ThreatEvent DB model to an API response schema. + """ + return ThreatEventResponse( + id=event.id, + created_at=event.created_at, + source_ip=event.source_ip, + request_method=event.request_method, + request_path=event.request_path, + status_code=event.status_code, + response_size=event.response_size, + user_agent=event.user_agent, + threat_score=event.threat_score, + severity=event.severity, # type: ignore[arg-type] + component_scores=event.component_scores, + geo=GeoInfo( + country=event.geo_country, + city=event.geo_city, + lat=event.geo_lat, + lon=event.geo_lon, + ), + matched_rules=event.matched_rules, + model_version=event.model_version, + reviewed=event.reviewed, + review_label=event.review_label, + ) + + +async def get_threats( + session: AsyncSession, + limit: int = 50, + offset: int = 0, + severity: str | None = None, + source_ip: str | None = None, + since: datetime | None = None, + until: datetime | None = None, +) -> ThreatListResponse: + """ + Query threat events with optional filters, returning a paginated response. + """ + query = select(ThreatEvent) + count_query = select(func.count()).select_from(ThreatEvent) + + if severity: + query = query.where(ThreatEvent.severity == severity.upper()) # type: ignore[arg-type] + count_query = count_query.where(ThreatEvent.severity == severity.upper()) # type: ignore[arg-type] + if source_ip: + query = query.where(ThreatEvent.source_ip == source_ip) # type: ignore[arg-type] + count_query = count_query.where(ThreatEvent.source_ip == source_ip) # type: ignore[arg-type] + if since: + query = query.where(ThreatEvent.created_at >= since) # type: ignore[arg-type] + count_query = count_query.where(ThreatEvent.created_at >= since) # type: ignore[arg-type] + if until: + query = query.where(ThreatEvent.created_at <= until) # type: ignore[arg-type] + count_query = count_query.where(ThreatEvent.created_at <= until) # type: ignore[arg-type] + + query = query.order_by(ThreatEvent.created_at.desc()) # type: ignore[attr-defined] + query = query.offset(offset).limit(limit) + + total = (await session.execute(count_query)).scalar_one() + rows = (await session.execute(query)).scalars().all() + + return ThreatListResponse( + total=total, + limit=limit, + offset=offset, + items=[_to_response(row) for row in rows], + ) + + +async def get_threat_by_id( + session: AsyncSession, + threat_id: uuid.UUID, +) -> ThreatEventResponse | None: + """ + Fetch a single threat event by its UUID. + """ + result = await session.get(ThreatEvent, threat_id) + if result is None: + return None + return _to_response(result) + + +async def create_threat_event( + session: AsyncSession, + scored: ScoredRequest, +) -> ThreatEvent: + """ + Persist a scored request as a threat event in the database. + """ + event = ThreatEvent( + source_ip=scored.entry.ip, + request_method=scored.entry.method, + request_path=scored.entry.path, + status_code=scored.entry.status_code, + response_size=scored.entry.response_size, + user_agent=scored.entry.user_agent, + threat_score=scored.rule_result.threat_score, + severity=scored.rule_result.severity, + component_scores=scored.rule_result.component_scores, + geo_country=scored.geo.country if scored.geo else None, + geo_city=scored.geo.city if scored.geo else None, + geo_lat=scored.geo.lat if scored.geo else None, + geo_lon=scored.geo.lon if scored.geo else None, + feature_vector=scored.feature_vector, + matched_rules=scored.rule_result.matched_rules or None, + model_version="rules-v1", + ) + session.add(event) + await session.flush() + return event diff --git a/PROJECTS/advanced/ai-threat-detection/backend/cli/main.py b/PROJECTS/advanced/ai-threat-detection/backend/cli/main.py new file mode 100644 index 00000000..a3d44c00 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/cli/main.py @@ -0,0 +1,93 @@ +""" +©AngelaMos | 2026 +main.py +""" + +import typer + +app = typer.Typer( + name="vigil", + help="AngelusVigil — AI-powered threat detection engine", + no_args_is_help=True, +) + + +@app.command() +def serve( + host: str = typer.Option("0.0.0.0", help="Bind address"), + port: int = typer.Option(8000, help="Bind port"), + reload: bool = typer.Option(False, help="Enable auto-reload for development"), +) -> None: + """ + Start the AngelusVigil API server. + """ + import uvicorn + + uvicorn.run( + "app.main:app", + host=host, + port=port, + reload=reload, + ) + + +@app.command() +def config() -> None: + """ + Print the current configuration (secrets redacted). + """ + from app.config import settings + + safe_fields = {} + for key, value in settings.model_dump().items(): + if any(secret in key for secret in ("key", "password", "secret", "token")): + safe_fields[key] = "***REDACTED***" + elif "url" in key and "@" in str(value): + safe_fields[key] = _redact_url(str(value)) + else: + safe_fields[key] = value + + for key, value in sorted(safe_fields.items()): + typer.echo(f" {key}: {value}") + + +@app.command() +def health( + url: str = typer.Option( + "http://localhost:8000", + help="Base URL of the running server", + ), +) -> None: + """ + Ping the running server's /health endpoint. + """ + import httpx + + try: + response = httpx.get(f"{url}/health", timeout=5.0) + response.raise_for_status() + data = response.json() + typer.echo(f" status: {data.get('status', 'unknown')}") + typer.echo(f" uptime: {data.get('uptime_seconds', 0):.0f}s") + typer.echo(f" pipeline: {'running' if data.get('pipeline_running') else 'stopped'}") + except httpx.ConnectError: + typer.echo("Error: cannot connect to server", err=True) + raise typer.Exit(code=1) from None + except httpx.HTTPStatusError as exc: + typer.echo(f"Error: server returned {exc.response.status_code}", err=True) + raise typer.Exit(code=1) from None + + +def _redact_url(url: str) -> str: + """ + Replace the user:password portion of a database URL with ***:***. + """ + if "://" not in url or "@" not in url: + return url + scheme, rest = url.split("://", 1) + _, host_part = rest.rsplit("@", 1) + return f"{scheme}://***:***@{host_part}" + + +if __name__ == "__main__": + app() diff --git a/PROJECTS/advanced/ai-threat-detection/backend/pyproject.toml b/PROJECTS/advanced/ai-threat-detection/backend/pyproject.toml index 625c5a57..e89ff1d5 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/pyproject.toml +++ b/PROJECTS/advanced/ai-threat-detection/backend/pyproject.toml @@ -157,7 +157,6 @@ load-plugins = [ "pylint_pydantic", ] persistent = true -suggestion-mode = true ignore = [ "venv", ".venv", @@ -198,20 +197,16 @@ disable = [ "W0718", "E0611", "E1101", + "C0415", + "E1102", + "R1732", ] [tool.pylint.design] -max-args = 7 +max-args = 12 max-attributes = 10 -max-locals = 30 -max-positional-arguments = 7 - -[tool.pylint."messages control"] -per-file-ignores = [ - "alembic/env.py:W0611", - "app/services/*:C0121", - "tests/*:W0212,W0621", -] +max-locals = 35 +max-positional-arguments = 12 [tool.pytest.ini_options] asyncio_mode = "auto" diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/conftest.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/conftest.py index 5eeee5e8..3a6bd74b 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/conftest.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/conftest.py @@ -3,9 +3,17 @@ conftest.py """ -import pytest +from collections.abc import AsyncIterator +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.pool import StaticPool +from sqlmodel import SQLModel + +from app.api.deps import get_session from app.config import Settings +from app.factory import create_app @pytest.fixture @@ -16,8 +24,65 @@ def test_settings() -> Settings: return Settings( env="testing", debug=False, - database_url="sqlite+aiosqlite:///test.db", + database_url="sqlite+aiosqlite://", redis_url="redis://localhost:6379/1", nginx_log_path="/tmp/test-access.log", geoip_db_path="/tmp/nonexistent.mmdb", ) + + +@pytest.fixture +async def db_engine(): + """ + In-memory SQLite engine shared across all sessions via StaticPool. + """ + from app.models.threat_event import ThreatEvent # noqa: F401 + + engine = create_async_engine( + "sqlite+aiosqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + async with engine.begin() as conn: + await conn.run_sync(SQLModel.metadata.create_all) + yield engine + await engine.dispose() + + +@pytest.fixture +async def db_session(db_engine) -> AsyncIterator[AsyncSession]: + """ + Async session bound to the shared in-memory DB. + """ + factory = async_sessionmaker( + db_engine, + class_=AsyncSession, + expire_on_commit=False, + ) + async with factory() as session: + yield session + + +@pytest.fixture +async def db_client(db_engine) -> AsyncIterator[AsyncClient]: + """ + HTTPX async client with DB dependency overridden to use in-memory SQLite. + """ + factory = async_sessionmaker( + db_engine, + class_=AsyncSession, + expire_on_commit=False, + ) + + test_app = create_app() + + async def override_get_session() -> AsyncIterator[AsyncSession]: + async with factory() as session: + yield session + await session.commit() + + test_app.dependency_overrides[get_session] = override_get_session + + transport = ASGITransport(app=test_app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + yield client diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_api.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_api.py index 5d03c93a..8441e262 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_api.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_api.py @@ -3,10 +3,14 @@ test_api.py """ +import uuid +from datetime import datetime, UTC + import pytest from httpx import ASGITransport, AsyncClient from app.main import app +from app.models.threat_event import ThreatEvent @pytest.mark.asyncio @@ -55,3 +59,147 @@ async def test_ready_returns_check_structure() -> None: assert "database" in data["checks"] assert "redis" in data["checks"] assert "models_loaded" in data["checks"] + + +@pytest.mark.asyncio +async def test_list_threats_empty(db_client) -> None: + """ + GET /threats on an empty database returns zero items. + """ + response = await db_client.get("/threats") + + assert response.status_code == 200 + data = response.json() + assert data["total"] == 0 + assert data["limit"] == 50 + assert data["offset"] == 0 + assert data["items"] == [] + + +@pytest.mark.asyncio +async def test_get_threat_not_found(db_client) -> None: + """ + GET /threats/{random_id} returns 404 when the event does not exist. + """ + fake_id = uuid.uuid4() + response = await db_client.get(f"/threats/{fake_id}") + + assert response.status_code == 404 + assert response.json()["detail"] == "Threat event not found" + + +@pytest.mark.asyncio +async def test_get_threat_by_id(db_session, db_client) -> None: + """ + Seed a threat event, then fetch it by ID. + """ + event_id = uuid.uuid4() + event = ThreatEvent( + id=event_id, + created_at=datetime.now(UTC), + source_ip="10.0.0.1", + request_method="GET", + request_path="/admin", + status_code=200, + response_size=512, + user_agent="TestBot/1.0", + threat_score=0.85, + severity="HIGH", + component_scores={"SQL_INJECTION": 0.85}, + feature_vector=[0.0] * 35, + matched_rules=["SQL_INJECTION"], + ) + db_session.add(event) + await db_session.commit() + + response = await db_client.get(f"/threats/{event_id}") + + assert response.status_code == 200 + data = response.json() + assert data["id"] == str(event_id) + assert data["source_ip"] == "10.0.0.1" + assert data["threat_score"] == 0.85 + assert data["severity"] == "HIGH" + assert data["matched_rules"] == ["SQL_INJECTION"] + assert data["geo"]["country"] is None + + +@pytest.mark.asyncio +async def test_list_threats_severity_filter(db_session, db_client) -> None: + """ + Seed threats with different severities and filter by HIGH. + """ + now = datetime.now(UTC) + for severity, score in [("HIGH", 0.9), ("MEDIUM", 0.6), ("LOW", 0.3)]: + event = ThreatEvent( + id=uuid.uuid4(), + created_at=now, + source_ip="192.168.1.1", + request_method="GET", + request_path="/test", + status_code=200, + response_size=100, + user_agent="Mozilla/5.0", + threat_score=score, + severity=severity, + component_scores={}, + feature_vector=[0.0] * 35, + ) + db_session.add(event) + await db_session.commit() + + response = await db_client.get("/threats", params={"severity": "HIGH"}) + + assert response.status_code == 200 + data = response.json() + assert data["total"] == 1 + assert data["items"][0]["severity"] == "HIGH" + + +@pytest.mark.asyncio +async def test_stats_empty_window(db_client) -> None: + """ + GET /stats on an empty database returns zero counts. + """ + response = await db_client.get("/stats") + + assert response.status_code == 200 + data = response.json() + assert data["time_range"] == "24h" + assert data["total_requests"] == 0 + assert data["threats_detected"] == 0 + assert data["severity_breakdown"]["high"] == 0 + assert data["severity_breakdown"]["medium"] == 0 + assert data["severity_breakdown"]["low"] == 0 + assert data["top_source_ips"] == [] + assert data["top_attacked_paths"] == [] + + +@pytest.mark.asyncio +async def test_model_status() -> None: + """ + GET /models/status returns rules-only detection mode. + """ + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/models/status") + + assert response.status_code == 200 + data = response.json() + assert data["detection_mode"] == "rules-only" + assert data["active_models"] == [] + + +@pytest.mark.asyncio +async def test_retrain_returns_202() -> None: + """ + POST /models/retrain returns 202 Accepted with a job ID. + """ + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.post("/models/retrain") + + assert response.status_code == 202 + data = response.json() + assert data["status"] == "accepted" + assert len(data["job_id"]) == 32 diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_detection.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_detection.py index e60961f9..87c24cfb 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_detection.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_detection.py @@ -3,9 +3,9 @@ test_detection.py """ -from datetime import datetime, timezone +from datetime import datetime, UTC -from app.core.detection.rules import RuleEngine, RuleResult +from app.core.detection.rules import RuleEngine from app.core.ingestion.parsers import ParsedLogEntry @@ -21,7 +21,7 @@ def _make_entry( """ return ParsedLogEntry( ip="93.184.216.34", - timestamp=datetime(2026, 2, 11, 14, 30, 0, tzinfo=timezone.utc), + timestamp=datetime(2026, 2, 11, 14, 30, 0, tzinfo=UTC), method=method, path=path, query_string=query_string, diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_features.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_features.py index fc2094fa..dcd496f5 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_features.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_features.py @@ -3,7 +3,7 @@ test_features.py """ -from datetime import datetime, timezone +from datetime import datetime, UTC from app.core.features.extractor import extract_request_features from app.core.ingestion.parsers import ParsedLogEntry @@ -50,7 +50,7 @@ def _make_entry( Build a ParsedLogEntry with sensible defaults for testing. """ if timestamp is None: - timestamp = datetime(2026, 2, 11, 14, 30, 0, tzinfo=timezone.utc) + timestamp = datetime(2026, 2, 11, 14, 30, 0, tzinfo=UTC) return ParsedLogEntry( ip=ip, @@ -80,21 +80,15 @@ def test_path_depth() -> None: """ assert extract_request_features(_make_entry(path="/"))["path_depth"] == 0 assert extract_request_features(_make_entry(path="/api"))["path_depth"] == 1 - assert extract_request_features( - _make_entry(path="/api/v1/users") - )["path_depth"] == 3 + assert extract_request_features(_make_entry(path="/api/v1/users"))["path_depth"] == 3 def test_path_entropy_high_vs_low() -> None: """ Random-character paths have higher entropy than simple paths. """ - low = extract_request_features( - _make_entry(path="/index.html") - )["path_entropy"] - high = extract_request_features( - _make_entry(path="/x8Kp2mQz7wR4vL1n") - )["path_entropy"] + low = extract_request_features(_make_entry(path="/index.html"))["path_entropy"] + high = extract_request_features(_make_entry(path="/x8Kp2mQz7wR4vL1n"))["path_entropy"] assert high > low @@ -102,9 +96,7 @@ def test_query_string_features() -> None: """ Query param count and length are extracted correctly. """ - features = extract_request_features( - _make_entry(query_string="page=1&sort=name&limit=50") - ) + features = extract_request_features(_make_entry(query_string="page=1&sort=name&limit=50")) assert features["query_param_count"] == 3 assert features["query_string_length"] == len("page=1&sort=name&limit=50") @@ -122,9 +114,7 @@ def test_url_encoding_detection() -> None: ) assert encoded["has_encoded_chars"] is True - clean = extract_request_features( - _make_entry(path="/index.html", query_string="") - ) + clean = extract_request_features(_make_entry(path="/index.html", query_string="")) assert clean["has_encoded_chars"] is False @@ -132,14 +122,10 @@ def test_double_encoding_detection() -> None: """ Double-encoded sequences like %2527 are flagged. """ - double = extract_request_features( - _make_entry(path="/path%2527trick") - ) + double = extract_request_features(_make_entry(path="/path%2527trick")) assert double["has_double_encoding"] is True - single = extract_request_features( - _make_entry(path="/path%27normal") - ) + single = extract_request_features(_make_entry(path="/path%27normal")) assert single["has_double_encoding"] is False @@ -147,33 +133,23 @@ def test_status_class() -> None: """ Status class groups status codes into Nxx buckets. """ - assert extract_request_features( - _make_entry(status_code=200) - )["status_class"] == "2xx" - assert extract_request_features( - _make_entry(status_code=404) - )["status_class"] == "4xx" - assert extract_request_features( - _make_entry(status_code=503) - )["status_class"] == "5xx" + assert extract_request_features(_make_entry(status_code=200))["status_class"] == "2xx" + assert extract_request_features(_make_entry(status_code=404))["status_class"] == "4xx" + assert extract_request_features(_make_entry(status_code=503))["status_class"] == "5xx" def test_temporal_features() -> None: """ Hour, day of week, and weekend flag derived from timestamp. """ - wednesday_2pm = datetime(2026, 2, 11, 14, 0, 0, tzinfo=timezone.utc) - features = extract_request_features( - _make_entry(timestamp=wednesday_2pm) - ) + wednesday_2pm = datetime(2026, 2, 11, 14, 0, 0, tzinfo=UTC) + features = extract_request_features(_make_entry(timestamp=wednesday_2pm)) assert features["hour_of_day"] == 14 assert features["day_of_week"] == 2 assert features["is_weekend"] is False - saturday_3am = datetime(2026, 2, 14, 3, 0, 0, tzinfo=timezone.utc) - weekend = extract_request_features( - _make_entry(timestamp=saturday_3am) - ) + saturday_3am = datetime(2026, 2, 14, 3, 0, 0, tzinfo=UTC) + weekend = extract_request_features(_make_entry(timestamp=saturday_3am)) assert weekend["hour_of_day"] == 3 assert weekend["day_of_week"] == 5 assert weekend["is_weekend"] is True @@ -200,14 +176,10 @@ def test_ua_scanner_detection() -> None: """ Known vulnerability scanner user agents are flagged. """ - nikto = extract_request_features( - _make_entry(user_agent="Mozilla/5.00 (Nikto/2.1.6)") - ) + nikto = extract_request_features(_make_entry(user_agent="Mozilla/5.00 (Nikto/2.1.6)")) assert nikto["is_known_scanner"] is True - sqlmap = extract_request_features( - _make_entry(user_agent="sqlmap/1.8") - ) + sqlmap = extract_request_features(_make_entry(user_agent="sqlmap/1.8")) assert sqlmap["is_known_scanner"] is True @@ -215,9 +187,7 @@ def test_attack_pattern_detection() -> None: """ SQLi, XSS, and path traversal patterns in paths are detected. """ - sqli = extract_request_features( - _make_entry(path="/users", query_string="id=1' OR 1=1--") - ) + sqli = extract_request_features(_make_entry(path="/users", query_string="id=1' OR 1=1--")) assert sqli["has_attack_pattern"] is True xss = extract_request_features( @@ -225,14 +195,10 @@ def test_attack_pattern_detection() -> None: ) assert xss["has_attack_pattern"] is True - traversal = extract_request_features( - _make_entry(path="/static/../../etc/passwd") - ) + traversal = extract_request_features(_make_entry(path="/static/../../etc/passwd")) assert traversal["has_attack_pattern"] is True - clean = extract_request_features( - _make_entry(path="/api/v1/health") - ) + clean = extract_request_features(_make_entry(path="/api/v1/health")) assert clean["has_attack_pattern"] is False @@ -240,13 +206,11 @@ def test_special_char_ratio() -> None: """ Paths with many non-alphanumeric characters have higher ratios. """ - clean = extract_request_features( - _make_entry(path="/api/users") - )["special_char_ratio"] + clean = extract_request_features(_make_entry(path="/api/users"))["special_char_ratio"] - noisy = extract_request_features( - _make_entry(path="/") - )["special_char_ratio"] + noisy = extract_request_features(_make_entry(path="/"))[ + "special_char_ratio" + ] assert noisy > clean @@ -255,51 +219,39 @@ def test_private_ip_detection() -> None: """ RFC 1918 and loopback addresses are classified as private. """ - assert extract_request_features( - _make_entry(ip="192.168.1.1") - )["is_private_ip"] is True + assert extract_request_features(_make_entry(ip="192.168.1.1"))["is_private_ip"] is True - assert extract_request_features( - _make_entry(ip="127.0.0.1") - )["is_private_ip"] is True + assert extract_request_features(_make_entry(ip="127.0.0.1"))["is_private_ip"] is True - assert extract_request_features( - _make_entry(ip="8.8.8.8") - )["is_private_ip"] is False + assert extract_request_features(_make_entry(ip="8.8.8.8"))["is_private_ip"] is False def test_file_extension() -> None: """ File extension is extracted from the path. """ - assert extract_request_features( - _make_entry(path="/style.css") - )["file_extension"] == ".css" + assert extract_request_features(_make_entry(path="/style.css"))["file_extension"] == ".css" - assert extract_request_features( - _make_entry(path="/api/users") - )["file_extension"] == "" + assert extract_request_features(_make_entry(path="/api/users"))["file_extension"] == "" def test_country_code_passthrough() -> None: """ Country code is passed through from the caller. """ - features = extract_request_features( - _make_entry(), country_code="US" - ) + features = extract_request_features(_make_entry(), country_code="US") assert features["country_code"] == "US" features_empty = extract_request_features(_make_entry()) assert features_empty["country_code"] == "" -import time +import time # noqa: E402 -import fakeredis.aioredis -import pytest +import fakeredis.aioredis # noqa: E402 +import pytest # noqa: E402 -from app.core.features.aggregator import WindowAggregator +from app.core.features.aggregator import WindowAggregator # noqa: E402 AGGREGATOR_KEYS = { "req_count_1m", @@ -476,8 +428,8 @@ async def test_aggregator_window_boundary(aggregator) -> None: assert result["req_count_5m"] == 2 -from app.core.features.encoder import encode_for_inference -from app.core.features.mappings import FEATURE_ORDER, METHOD_MAP, STATUS_CLASS_MAP +from app.core.features.encoder import encode_for_inference # noqa: E402 +from app.core.features.mappings import FEATURE_ORDER, METHOD_MAP, STATUS_CLASS_MAP # noqa: E402 def _full_features() -> dict[str, int | float | bool | str]: diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_geoip.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_geoip.py index e2d22d03..00a0fa43 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_geoip.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_geoip.py @@ -3,7 +3,6 @@ test_geoip.py """ -import asyncio from unittest.mock import MagicMock, patch import pytest @@ -109,9 +108,7 @@ async def test_lookup_missing_city_name(mock_reader) -> None: """ Responses with None city name are handled gracefully. """ - mock_reader.city.return_value = _mock_city_response( - city_name=None, lat=0.0, lon=0.0 - ) + mock_reader.city.return_value = _mock_city_response(city_name=None, lat=0.0, lon=0.0) service = GeoIPService.__new__(GeoIPService) service._reader = mock_reader diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_integration.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_integration.py new file mode 100644 index 00000000..28093b50 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_integration.py @@ -0,0 +1,199 @@ +""" +©AngelaMos | 2026 +test_integration.py +""" + +import asyncio +import os +import shutil +import tempfile +from pathlib import Path + +import pytest +from fakeredis.aioredis import FakeRedis +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.pool import StaticPool +from sqlmodel import SQLModel + +from app.core.alerts.dispatcher import AlertDispatcher +from app.core.detection.rules import RuleEngine +from app.core.ingestion.pipeline import Pipeline +from app.core.ingestion.tailer import LogTailer +from app.models.threat_event import ThreatEvent + +NORMAL_LINE = ( + "192.168.1.100 - - [11/Feb/2026:10:00:00 +0000] " + '"GET /index.html HTTP/1.1" 200 4523 "-" ' + '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' +) + +SQLI_LINE = ( + "198.51.100.10 - - [11/Feb/2026:10:00:01 +0000] " + '"GET /search?q=1%27+OR+1=1-- HTTP/1.1" 200 5678 "-" ' + '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' +) + +XSS_LINE = ( + "198.51.100.11 - - [11/Feb/2026:10:00:02 +0000] " + '"GET /comment?text= HTTP/1.1" 200 3210 "-" ' + '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' +) + +PATH_TRAVERSAL_LINE = ( + "198.51.100.12 - - [11/Feb/2026:10:00:03 +0000] " + '"GET /../../etc/passwd HTTP/1.1" 400 230 "-" ' + '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' +) + + +def _write_lines(log_path: str, *lines: str) -> None: + """ + Append lines to the log file and force an OS-level flush + so inotify fires immediately. + """ + with open(log_path, "a") as f: + for line in lines: + f.write(line + "\n") + f.flush() + os.fsync(f.fileno()) + + +@pytest.fixture +async def integration_env(): + """ + Full-stack integration environment with in-memory DB, + fake Redis, pipeline, and temp log directory. + """ + + tmp_dir = tempfile.mkdtemp() + log_path = os.path.join(tmp_dir, "access.log") + Path(log_path).touch() + + fake_redis = FakeRedis(decode_responses=True) + engine = create_async_engine( + "sqlite+aiosqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + async with engine.begin() as conn: + await conn.run_sync(SQLModel.metadata.create_all) + + session_factory = async_sessionmaker( + engine, + class_=AsyncSession, + expire_on_commit=False, + ) + + dispatcher = AlertDispatcher(fake_redis, session_factory) + rule_engine = RuleEngine() + + pipeline = Pipeline( + redis_client=fake_redis, + rule_engine=rule_engine, + on_result=dispatcher.dispatch, + ) + await pipeline.start() + + loop = asyncio.get_running_loop() + tailer = LogTailer(log_path, pipeline.raw_queue, loop) + tailer.start() + + await asyncio.sleep(0.5) + + yield { + "log_path": log_path, + "pipeline": pipeline, + "tailer": tailer, + "session_factory": session_factory, + "engine": engine, + } + + tailer.stop() + await pipeline.stop() + await engine.dispose() + shutil.rmtree(tmp_dir, ignore_errors=True) + + +async def _poll_threat_count( + session_factory: async_sessionmaker[AsyncSession], + expected: int, + timeout: float = 8.0, +) -> int: + """ + Poll the database until the expected threat count is reached or timeout. + """ + count = 0 + for _ in range(int(timeout / 0.1)): + await asyncio.sleep(0.1) + async with session_factory() as session: + result = await session.execute(select(func.count()).select_from(ThreatEvent)) + count = result.scalar_one() + if count >= expected: + return count + return count + + +@pytest.mark.asyncio +async def test_tailer_to_db_end_to_end(integration_env) -> None: + """ + Write log lines to a file - tailer picks them up - pipeline processes - + dispatcher stores MEDIUM+ threats in the database. + """ + env = integration_env + _write_lines( + env["log_path"], + NORMAL_LINE, + SQLI_LINE, + XSS_LINE, + PATH_TRAVERSAL_LINE, + ) + + count = await _poll_threat_count(env["session_factory"], expected=3) + assert count >= 3, f"Expected >= 3 stored threats, got {count}" + + +@pytest.mark.asyncio +async def test_only_medium_plus_stored(integration_env) -> None: + """ + Normal (LOW severity) requests are NOT stored in the database. + Only MEDIUM and HIGH severity threats are persisted. + """ + env = integration_env + lines = [ + f"192.168.1.{i + 1} - - [11/Feb/2026:10:00:0{i} +0000] " + f'"GET /page/{i} HTTP/1.1" 200 1234 "-" ' + f'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' + for i in range(5) + ] + _write_lines(env["log_path"], *lines) + + await asyncio.sleep(2.0) + + async with env["session_factory"]() as session: + result = await session.execute(select(func.count()).select_from(ThreatEvent)) + count = result.scalar_one() + + assert count == 0 + + +@pytest.mark.asyncio +async def test_stored_threats_have_correct_fields(integration_env) -> None: + """ + Stored threat events have populated severity, score, and matched rules. + """ + env = integration_env + _write_lines(env["log_path"], SQLI_LINE) + + count = await _poll_threat_count(env["session_factory"], expected=1) + assert count >= 1, f"Expected >= 1 stored threat, got {count}" + + async with env["session_factory"]() as session: + rows = (await session.execute(select(ThreatEvent))).scalars().all() + + event = rows[0] + assert event.severity in ("HIGH", "MEDIUM") + assert event.threat_score >= 0.5 + assert len(event.matched_rules) > 0 + assert len(event.feature_vector) == 35 + assert event.source_ip == "198.51.100.10" diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_parsers.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_parsers.py index 59767582..fa5d031a 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_parsers.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_parsers.py @@ -3,7 +3,7 @@ test_parsers.py """ -from datetime import datetime, timezone +from datetime import datetime, UTC from app.core.ingestion.parsers import ParsedLogEntry, parse_combined @@ -13,7 +13,7 @@ def test_parse_standard_combined_line() -> None: Parse a standard nginx combined log line into all fields. """ line = ( - '93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] ' + "93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] " '"GET /api/users?page=1 HTTP/1.1" 200 1234 ' '"https://example.com" ' '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' @@ -23,7 +23,7 @@ def test_parse_standard_combined_line() -> None: assert result is not None assert isinstance(result, ParsedLogEntry) assert result.ip == "93.184.216.34" - assert result.timestamp == datetime(2026, 2, 11, 14, 30, 0, tzinfo=timezone.utc) + assert result.timestamp == datetime(2026, 2, 11, 14, 30, 0, tzinfo=UTC) assert result.method == "GET" assert result.path == "/api/users" assert result.query_string == "page=1" @@ -38,10 +38,7 @@ def test_parse_ipv6_address() -> None: """ Parse a line with an IPv6 source address. """ - line = ( - '::1 - - [11/Feb/2026:14:30:00 +0000] ' - '"GET / HTTP/1.1" 200 612 "-" "curl/8.0"' - ) + line = '::1 - - [11/Feb/2026:14:30:00 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/8.0"' result = parse_combined(line) assert result is not None @@ -56,7 +53,7 @@ def test_parse_missing_referer() -> None: A dash referer is normalized to an empty string. """ line = ( - '10.0.0.1 - - [11/Feb/2026:08:15:42 +0000] ' + "10.0.0.1 - - [11/Feb/2026:08:15:42 +0000] " '"POST /login HTTP/1.1" 302 0 "-" "Mozilla/5.0"' ) result = parse_combined(line) @@ -72,7 +69,7 @@ def test_parse_complex_query_string() -> None: Query strings with multiple parameters and special characters. """ line = ( - '93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] ' + "93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] " '"GET /search?q=hello+world&lang=en&page=2&sort=relevance HTTP/1.1" ' '200 5678 "https://example.com/search" "Mozilla/5.0"' ) @@ -101,10 +98,7 @@ def test_parse_dash_response_size() -> None: """ A dash response size (e.g. HEAD 304) is normalized to zero. """ - line = ( - '1.2.3.4 - - [11/Feb/2026:10:00:00 +0000] ' - '"HEAD / HTTP/1.1" 304 - "-" "Mozilla/5.0"' - ) + line = '1.2.3.4 - - [11/Feb/2026:10:00:00 +0000] "HEAD / HTTP/1.1" 304 - "-" "Mozilla/5.0"' result = parse_combined(line) assert result is not None @@ -117,7 +111,7 @@ def test_parse_full_ipv6_address() -> None: Parse a line with a full-length IPv6 address. """ line = ( - '2001:0db8:85a3:0000:0000:8a2e:0370:7334 - - ' + "2001:0db8:85a3:0000:0000:8a2e:0370:7334 - - " '[11/Feb/2026:14:30:00 +0000] "GET /api/v1/health HTTP/2.0" ' '200 256 "-" "python-httpx/0.28"' ) diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_pipeline.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_pipeline.py index c2b76b86..e294251e 100644 --- a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_pipeline.py +++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_pipeline.py @@ -3,8 +3,6 @@ test_pipeline.py """ -import asyncio - import fakeredis.aioredis import pytest @@ -12,14 +10,14 @@ from app.core.detection.rules import RuleEngine from app.core.ingestion.pipeline import Pipeline, ScoredRequest VALID_LINE = ( - '93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] ' + "93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] " '"GET /api/v1/users HTTP/1.1" 200 1234 ' '"https://example.com" ' '"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"' ) SQLI_LINE = ( - '93.184.216.34 - - [11/Feb/2026:14:30:01 +0000] ' + "93.184.216.34 - - [11/Feb/2026:14:30:01 +0000] " '"GET /users?id=1%27OR%201=1-- HTTP/1.1" 200 512 ' '"-" "Mozilla/5.0"' ) diff --git a/PROJECTS/advanced/ai-threat-detection/data/sample-logs/sample-access.log b/PROJECTS/advanced/ai-threat-detection/data/sample-logs/sample-access.log new file mode 100644 index 00000000..cf88e648 --- /dev/null +++ b/PROJECTS/advanced/ai-threat-detection/data/sample-logs/sample-access.log @@ -0,0 +1,50 @@ +192.168.1.10 - - [11/Feb/2026:08:00:01 +0000] "GET /index.html HTTP/1.1" 200 4523 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +192.168.1.10 - - [11/Feb/2026:08:00:02 +0000] "GET /static/css/main.css HTTP/1.1" 200 8921 "http://example.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +192.168.1.10 - - [11/Feb/2026:08:00:02 +0000] "GET /static/js/app.js HTTP/1.1" 200 45230 "http://example.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +192.168.1.10 - - [11/Feb/2026:08:00:03 +0000] "GET /static/img/logo.png HTTP/1.1" 304 0 "http://example.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +10.0.0.50 - - [11/Feb/2026:08:01:00 +0000] "GET /api/v1/users HTTP/1.1" 200 2340 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" +10.0.0.50 - - [11/Feb/2026:08:01:01 +0000] "POST /api/v1/auth/login HTTP/1.1" 200 312 "http://example.com/login" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" +10.0.0.50 - - [11/Feb/2026:08:01:02 +0000] "GET /api/v1/dashboard HTTP/1.1" 200 8920 "http://example.com/dashboard" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" +172.16.0.5 - - [11/Feb/2026:08:02:00 +0000] "GET / HTTP/1.1" 301 162 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0" +172.16.0.5 - - [11/Feb/2026:08:02:01 +0000] "GET /index.html HTTP/1.1" 200 4523 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0" +172.16.0.5 - - [11/Feb/2026:08:02:05 +0000] "GET /about HTTP/1.1" 200 3210 "http://example.com/" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0" +172.16.0.5 - - [11/Feb/2026:08:02:06 +0000] "GET /contact HTTP/1.1" 200 2100 "http://example.com/about" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0" +192.168.1.20 - - [11/Feb/2026:08:03:00 +0000] "GET /products HTTP/1.1" 200 12450 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1" +192.168.1.20 - - [11/Feb/2026:08:03:01 +0000] "GET /products/42 HTTP/1.1" 200 3210 "http://example.com/products" "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1" +192.168.1.20 - - [11/Feb/2026:08:03:02 +0000] "POST /api/v1/cart HTTP/1.1" 201 128 "http://example.com/products/42" "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1" +192.168.1.20 - - [11/Feb/2026:08:03:05 +0000] "GET /api/v1/cart HTTP/1.1" 200 456 "http://example.com/cart" "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1" +10.0.0.100 - - [11/Feb/2026:08:04:00 +0000] "GET /robots.txt HTTP/1.1" 200 230 "-" "Googlebot/2.1 (+http://www.google.com/bot.html)" +10.0.0.100 - - [11/Feb/2026:08:04:01 +0000] "GET /sitemap.xml HTTP/1.1" 200 9800 "-" "Googlebot/2.1 (+http://www.google.com/bot.html)" +10.0.0.101 - - [11/Feb/2026:08:05:00 +0000] "GET / HTTP/1.1" 200 4523 "-" "Bingbot/2.0 (+http://www.bing.com/bingbot.htm)" +10.0.0.102 - - [11/Feb/2026:08:06:00 +0000] "HEAD / HTTP/1.1" 200 0 "-" "Mozilla/5.0 (compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)" +172.16.0.10 - - [11/Feb/2026:08:07:00 +0000] "GET /api/v1/health HTTP/1.1" 200 89 "-" "curl/8.7.1" +172.16.0.10 - - [11/Feb/2026:08:07:01 +0000] "GET /api/v1/ready HTTP/1.1" 200 145 "-" "curl/8.7.1" +192.168.1.30 - - [11/Feb/2026:08:08:00 +0000] "GET /blog HTTP/1.1" 200 15600 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +192.168.1.30 - - [11/Feb/2026:08:08:02 +0000] "GET /blog/post/hello-world HTTP/1.1" 200 8900 "http://example.com/blog" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +192.168.1.30 - - [11/Feb/2026:08:08:05 +0000] "POST /blog/post/hello-world/comment HTTP/1.1" 201 256 "http://example.com/blog/post/hello-world" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +10.0.0.55 - - [11/Feb/2026:08:09:00 +0000] "GET /docs HTTP/1.1" 200 23400 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +10.0.0.55 - - [11/Feb/2026:08:09:01 +0000] "GET /docs/api HTTP/1.1" 200 12300 "http://example.com/docs" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +10.0.0.55 - - [11/Feb/2026:08:09:02 +0000] "GET /docs/quickstart HTTP/1.1" 200 9800 "http://example.com/docs" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +192.168.1.40 - - [11/Feb/2026:08:10:00 +0000] "GET /favicon.ico HTTP/1.1" 200 1150 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +192.168.1.40 - - [11/Feb/2026:08:10:01 +0000] "GET /static/fonts/roboto.woff2 HTTP/1.1" 200 45000 "http://example.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +172.16.0.20 - - [11/Feb/2026:08:11:00 +0000] "GET /nonexistent-page HTTP/1.1" 404 1234 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0" +172.16.0.20 - - [11/Feb/2026:08:11:01 +0000] "GET /old-page HTTP/1.1" 301 162 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0" +10.0.0.60 - - [11/Feb/2026:08:12:00 +0000] "OPTIONS /api/v1/users HTTP/1.1" 204 0 "http://frontend.example.com" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +10.0.0.60 - - [11/Feb/2026:08:12:01 +0000] "PUT /api/v1/users/5 HTTP/1.1" 200 512 "http://frontend.example.com/settings" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +10.0.0.60 - - [11/Feb/2026:08:12:02 +0000] "DELETE /api/v1/cart/item/7 HTTP/1.1" 204 0 "http://frontend.example.com/cart" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +192.168.1.50 - - [11/Feb/2026:08:13:00 +0000] "GET /search?q=wireless+keyboard HTTP/1.1" 200 6700 "http://example.com/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" +192.168.1.50 - - [11/Feb/2026:08:13:01 +0000] "GET /search?q=usb+hub&page=2 HTTP/1.1" 200 5400 "http://example.com/search?q=wireless+keyboard" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" +10.0.0.70 - - [11/Feb/2026:08:14:00 +0000] "GET /api/v1/orders HTTP/1.1" 401 89 "-" "python-requests/2.32.0" +10.0.0.70 - - [11/Feb/2026:08:14:01 +0000] "POST /api/v1/auth/token HTTP/1.1" 200 256 "-" "python-requests/2.32.0" +10.0.0.70 - - [11/Feb/2026:08:14:02 +0000] "GET /api/v1/orders HTTP/1.1" 200 3400 "-" "python-requests/2.32.0" +192.168.1.60 - - [11/Feb/2026:08:15:00 +0000] "GET /static/img/hero.jpg HTTP/1.1" 200 250000 "http://example.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +203.0.113.50 - - [11/Feb/2026:08:20:00 +0000] "GET / HTTP/1.1" 200 4523 "-" "Nikto/2.1.6" +203.0.113.50 - - [11/Feb/2026:08:20:01 +0000] "GET /admin HTTP/1.1" 404 1234 "-" "Nikto/2.1.6" +203.0.113.50 - - [11/Feb/2026:08:20:02 +0000] "GET /phpmyadmin HTTP/1.1" 404 1234 "-" "Nikto/2.1.6" +203.0.113.51 - - [11/Feb/2026:08:21:00 +0000] "GET /.env HTTP/1.1" 404 1234 "-" "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)" +203.0.113.51 - - [11/Feb/2026:08:21:01 +0000] "GET /wp-login.php HTTP/1.1" 404 1234 "-" "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)" +198.51.100.10 - - [11/Feb/2026:08:30:00 +0000] "GET /search?q=1%27+OR+1=1-- HTTP/1.1" 200 5678 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +198.51.100.11 - - [11/Feb/2026:08:31:00 +0000] "GET /comment?text= HTTP/1.1" 200 3210 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +198.51.100.12 - - [11/Feb/2026:08:32:00 +0000] "GET /../../etc/passwd HTTP/1.1" 400 230 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +198.51.100.13 - - [11/Feb/2026:08:33:00 +0000] "GET /api/ping?host=;cat /etc/passwd HTTP/1.1" 200 1200 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +198.51.100.14 - - [11/Feb/2026:08:34:00 +0000] "GET /page?file=php://filter/convert.base64-encode/resource=index HTTP/1.1" 200 8900 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" diff --git a/PROJECTS/advanced/ai-threat-detection/dev.compose.yml b/PROJECTS/advanced/ai-threat-detection/dev.compose.yml index 4fc2457a..5f3a12c7 100644 --- a/PROJECTS/advanced/ai-threat-detection/dev.compose.yml +++ b/PROJECTS/advanced/ai-threat-detection/dev.compose.yml @@ -1,5 +1,5 @@ # ©AngelaMos | 2026 -# Development Docker Compose +# dev.compose.yml services: postgres: @@ -8,90 +8,74 @@ services: environment: POSTGRES_DB: ${POSTGRES_DB:-angelusvigil} POSTGRES_USER: ${POSTGRES_USER:-vigil} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD required} - POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" - volumes: - - postgres_dev_data:/var/lib/postgresql/data + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-devpassword} ports: - "${POSTGRES_HOST_PORT:-16969}:5432" + volumes: + - postgres_dev:/var/lib/postgresql/data networks: - - vigil_network_dev + - vigil_dev healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-vigil} -d ${POSTGRES_DB:-angelusvigil}"] - interval: 10s - timeout: 5s + interval: 5s + timeout: 3s retries: 5 + start_period: 10s redis: image: redis:7.4-alpine container_name: vigil-redis-dev - command: redis-server /usr/local/etc/redis/redis.conf - volumes: - - redis_dev_data:/data - - ./infra/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro + command: redis-server --appendonly yes ports: - "${REDIS_HOST_PORT:-26969}:6379" + volumes: + - redis_dev:/data networks: - - vigil_network_dev + - vigil_dev healthcheck: test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 5s + interval: 5s + timeout: 3s retries: 5 + start_period: 3s backend: build: context: . - dockerfile: infra/docker/fastapi.dev + dockerfile: infra/docker/fastapi.prod container_name: vigil-backend-dev environment: - ENV: ${ENV:-development} - DEBUG: ${DEBUG:-true} - LOG_LEVEL: ${LOG_LEVEL:-DEBUG} - API_KEY: ${API_KEY:-dev-api-key} - DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-vigil}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-angelusvigil} - REDIS_URL: ${REDIS_URL:-redis://redis:6379} + ENV: development + DEBUG: "true" + LOG_LEVEL: INFO + API_KEY: ${API_KEY:-dev-test-key} + DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-vigil}:${POSTGRES_PASSWORD:-devpassword}@postgres:5432/${POSTGRES_DB:-angelusvigil} + REDIS_URL: redis://redis:6379 NGINX_LOG_PATH: /var/log/nginx/access.log GEOIP_DB_PATH: /usr/share/GeoIP/GeoLite2-City.mmdb + ports: + - "${BACKEND_HOST_PORT:-36969}:8000" volumes: - - ./backend:/app - - /app/.venv - - /app/__pycache__ - - ./data/sample-logs:/var/log/nginx:ro - - geoip_dev_data:/usr/share/GeoIP:ro + - nginx_logs_dev:/var/log/nginx depends_on: postgres: condition: service_healthy redis: condition: service_healthy networks: - - vigil_network_dev - ports: - - "${BACKEND_HOST_PORT:-36969}:8000" - restart: unless-stopped - - geoip-updater: - image: maxmindinc/geoipupdate:latest - container_name: vigil-geoip-dev - environment: - GEOIPUPDATE_ACCOUNT_ID: ${GEOIP_ACCOUNT_ID:-0} - GEOIPUPDATE_LICENSE_KEY: ${GEOIP_LICENSE_KEY:-placeholder} - GEOIPUPDATE_EDITION_IDS: "GeoLite2-City" - GEOIPUPDATE_FREQUENCY: "168" - volumes: - - geoip_dev_data:/usr/share/GeoIP - networks: - - vigil_network_dev + - vigil_dev + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 15s networks: - vigil_network_dev: + vigil_dev: driver: bridge - name: vigil_network_dev volumes: - postgres_dev_data: - name: vigil_postgres_dev_data - redis_dev_data: - name: vigil_redis_dev_data - geoip_dev_data: - name: vigil_geoip_dev_data + postgres_dev: + redis_dev: + nginx_logs_dev: diff --git a/PROJECTS/advanced/ai-threat-detection/infra/docker/fastapi.prod b/PROJECTS/advanced/ai-threat-detection/infra/docker/fastapi.prod index 6bd6233e..5813ceea 100644 --- a/PROJECTS/advanced/ai-threat-detection/infra/docker/fastapi.prod +++ b/PROJECTS/advanced/ai-threat-detection/infra/docker/fastapi.prod @@ -18,7 +18,8 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* COPY backend/pyproject.toml ./ -RUN uv pip install --target /app/deps . +RUN uv pip compile pyproject.toml -o requirements.txt && \ + uv pip install --target /app/deps -r requirements.txt FROM python:3.14-slim