feat: complete Phase 1 — rule-based detection pipeline, API, Docker E2E
Phase 1 delivers end-to-end threat detection from nginx log ingestion through rule-based scoring to REST API and WebSocket alerts. - API layer: /threats, /stats, /models/status, /ws/alerts, /health, /ready - Alert dispatcher with Redis pub/sub + PostgreSQL persistence - Typer CLI: vigil serve, config, health - Integration tests (tailer -> pipeline -> DB round-trip) - Docker E2E verified: dev.compose.yml stack with all services healthy - Linting clean: ruff, mypy strict (0 issues), pylint 10/10 - 75 tests passing across 8 test modules
This commit is contained in:
parent
3fd9f77335
commit
7f35fbd839
|
|
@ -3,4 +3,4 @@
|
|||
*.cache
|
||||
*.coverage
|
||||
.env
|
||||
|
||||
.angelusvigil/
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
# ©AngelaMos | 2026
|
||||
# .dockerignore
|
||||
|
||||
.git
|
||||
.gitignore
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
.angelusvigil/
|
||||
docs/
|
||||
|
||||
backend/.venv/
|
||||
backend/.pytest_cache/
|
||||
backend/.mypy_cache/
|
||||
backend/.ruff_cache/
|
||||
backend/__pycache__/
|
||||
backend/**/__pycache__/
|
||||
backend/htmlcov/
|
||||
backend/.coverage
|
||||
backend/*.egg-info/
|
||||
|
||||
*.log
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
.DS_Store
|
||||
|
|
@ -66,6 +66,7 @@ docker-compose.override.yml
|
|||
data/models/
|
||||
data/geoip/
|
||||
data/sample-logs/*.log
|
||||
!data/sample-logs/sample-access.log
|
||||
|
||||
# MLflow
|
||||
mlruns/
|
||||
|
|
|
|||
|
|
@ -1,11 +1,20 @@
|
|||
# AngelusVigil
|
||||
|
||||
IN PROGRESS
|
||||
> **Status: IN PROGRESS** — Phase 1 complete, Phase 2 (ML models) next
|
||||
|
||||
AI-powered threat detection engine that analyzes web server access logs using machine learning to classify HTTP traffic as benign or malicious in real-time.
|
||||
|
||||
Deploys as a Docker sidecar alongside any nginx-based infrastructure. Zero code changes to the monitored application.
|
||||
|
||||
## Progress
|
||||
|
||||
| Phase | Description | Status |
|
||||
|-------|-------------|--------|
|
||||
| Phase 1 | Core pipeline, rule-based detection, API, Docker | Complete |
|
||||
| Phase 2 | ML ensemble (autoencoder + RF + IF), ONNX inference | Next |
|
||||
| Phase 3 | Production hardening, monitoring, retraining | Planned |
|
||||
| Phase 4 | Dashboard, active learning, explainability | Planned |
|
||||
|
||||
## Tech Stack
|
||||
|
||||
| Layer | Technology |
|
||||
|
|
@ -20,8 +29,8 @@ Deploys as a Docker sidecar alongside any nginx-based infrastructure. Zero code
|
|||
## Quick Start
|
||||
|
||||
```bash
|
||||
just setup
|
||||
just dev-up
|
||||
docker compose -f dev.compose.yml up -d
|
||||
curl http://localhost:36969/health
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
|
@ -32,6 +41,8 @@ just dev-up
|
|||
- **MEDIUM** (0.5-0.7): Store + monitor
|
||||
- **LOW** (<0.5): Log only
|
||||
|
||||
Currently running rule-based detection (ModSecurity CRS patterns) as cold-start fallback until ML models are trained in Phase 2.
|
||||
|
||||
See `learn/` for detailed documentation.
|
||||
|
||||
## License
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
deps.py
|
||||
"""
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from fastapi import Request
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
||||
async def get_session(request: Request) -> AsyncIterator[AsyncSession]:
|
||||
"""
|
||||
Yield an async database session from
|
||||
the application's session factory
|
||||
"""
|
||||
factory = request.app.state.session_factory
|
||||
async with factory() as session:
|
||||
yield session
|
||||
await session.commit()
|
||||
|
|
@ -6,6 +6,7 @@ health.py
|
|||
import time
|
||||
|
||||
from fastapi import APIRouter, Request, Response
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.core.redis_manager import redis_manager
|
||||
|
||||
|
|
@ -13,7 +14,7 @@ router = APIRouter()
|
|||
|
||||
|
||||
@router.get("/health")
|
||||
async def health(request: Request) -> dict:
|
||||
async def health(request: Request) -> dict[str, object]:
|
||||
"""
|
||||
Liveness probe — returns 200 if the process is alive.
|
||||
"""
|
||||
|
|
@ -26,7 +27,7 @@ async def health(request: Request) -> dict:
|
|||
|
||||
|
||||
@router.get("/ready")
|
||||
async def ready(request: Request, response: Response) -> dict:
|
||||
async def ready(request: Request, response: Response) -> dict[str, object]:
|
||||
"""
|
||||
Readiness probe — checks all service dependencies.
|
||||
"""
|
||||
|
|
@ -66,7 +67,7 @@ async def _check_database(request: Request) -> bool:
|
|||
return False
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
await conn.execute("SELECT 1")
|
||||
await conn.execute(text("SELECT 1"))
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
models_api.py
|
||||
"""
|
||||
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
router = APIRouter(prefix="/models", tags=["models"])
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def model_status() -> dict[str, object]:
|
||||
"""
|
||||
Return the status of active ML models.
|
||||
"""
|
||||
return {
|
||||
"active_models": [],
|
||||
"detection_mode": "rules-only",
|
||||
"note": "ML models available after Phase 2 training",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/retrain", status_code=202)
|
||||
async def retrain() -> dict[str, object]:
|
||||
"""
|
||||
Trigger an async model retraining job.
|
||||
"""
|
||||
return {
|
||||
"status": "accepted",
|
||||
"job_id": uuid.uuid4().hex,
|
||||
"note": "Retraining not available in Phase 1",
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
stats.py
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_session
|
||||
from app.schemas.stats import StatsResponse
|
||||
from app.services import stats_service
|
||||
|
||||
router = APIRouter(prefix="/stats", tags=["stats"])
|
||||
|
||||
|
||||
@router.get("", response_model=StatsResponse)
|
||||
async def get_stats(
|
||||
session: AsyncSession = Depends(get_session),
|
||||
time_range: str = Query("24h", alias="range"),
|
||||
) -> StatsResponse:
|
||||
"""
|
||||
Aggregate threat statistics for a given time window.
|
||||
"""
|
||||
return await stats_service.get_stats(session, time_range)
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
threats.py
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_session
|
||||
from app.schemas.threats import ThreatEventResponse, ThreatListResponse
|
||||
from app.services import threat_service
|
||||
|
||||
router = APIRouter(prefix="/threats", tags=["threats"])
|
||||
|
||||
|
||||
@router.get("", response_model=ThreatListResponse)
|
||||
async def list_threats(
|
||||
session: AsyncSession = Depends(get_session),
|
||||
limit: int = Query(50, ge=1, le=100),
|
||||
offset: int = Query(0, ge=0),
|
||||
severity: str | None = Query(None),
|
||||
source_ip: str | None = Query(None),
|
||||
since: datetime | None = Query(None),
|
||||
until: datetime | None = Query(None),
|
||||
) -> ThreatListResponse:
|
||||
"""
|
||||
List threat events with optional filters and pagination.
|
||||
"""
|
||||
return await threat_service.get_threats(
|
||||
session,
|
||||
limit,
|
||||
offset,
|
||||
severity,
|
||||
source_ip,
|
||||
since,
|
||||
until,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{threat_id}", response_model=ThreatEventResponse)
|
||||
async def get_threat(
|
||||
threat_id: uuid.UUID,
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> ThreatEventResponse:
|
||||
"""
|
||||
Fetch a single threat event by ID.
|
||||
"""
|
||||
result = await threat_service.get_threat_by_id(session, threat_id)
|
||||
if result is None:
|
||||
raise HTTPException(status_code=404, detail="Threat event not found")
|
||||
return result
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
websocket.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
|
||||
|
||||
from app.core.alerts import ALERTS_CHANNEL
|
||||
from app.core.redis_manager import redis_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.websocket("/ws/alerts")
|
||||
async def ws_alerts(websocket: WebSocket) -> None:
|
||||
"""
|
||||
Stream real time threat alerts to connected WebSocket clients
|
||||
via Redis pub/sub relay.
|
||||
|
||||
Each client gets its own Redis subscriber so this works correctly
|
||||
across multiple FastAPI workers.
|
||||
"""
|
||||
await websocket.accept()
|
||||
|
||||
redis = redis_manager.client
|
||||
if redis is None:
|
||||
await websocket.close(code=1011, reason="Redis not available")
|
||||
return
|
||||
|
||||
pubsub = redis.pubsub()
|
||||
await pubsub.subscribe(ALERTS_CHANNEL)
|
||||
|
||||
async def _relay() -> None:
|
||||
async for message in pubsub.listen():
|
||||
if message["type"] == "message":
|
||||
await websocket.send_text(message["data"])
|
||||
|
||||
async def _receive() -> None:
|
||||
try:
|
||||
while True:
|
||||
await websocket.receive()
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
|
||||
relay_task = asyncio.create_task(_relay())
|
||||
receive_task = asyncio.create_task(_receive())
|
||||
|
||||
try:
|
||||
done, pending = await asyncio.wait(
|
||||
[relay_task, receive_task],
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
for task in pending:
|
||||
task.cancel()
|
||||
finally:
|
||||
await pubsub.unsubscribe(ALERTS_CHANNEL)
|
||||
await pubsub.aclose() # type: ignore[attr-defined]
|
||||
logger.debug("WebSocket client disconnected")
|
||||
|
|
@ -25,9 +25,7 @@ class Settings(BaseSettings):
|
|||
api_key: str = ""
|
||||
log_level: str = "INFO"
|
||||
|
||||
database_url: str = (
|
||||
"postgresql+asyncpg://vigil:changeme@localhost:5432/angelusvigil"
|
||||
)
|
||||
database_url: str = "postgresql+asyncpg://vigil:changeme@localhost:5432/angelusvigil"
|
||||
|
||||
redis_url: str = "redis://localhost:6379"
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
||||
ALERTS_CHANNEL = "alerts"
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
dispatcher.py
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import redis.asyncio as aioredis
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from app.core.alerts import ALERTS_CHANNEL
|
||||
from app.core.ingestion.pipeline import ScoredRequest
|
||||
from app.schemas.websocket import WebSocketAlert
|
||||
from app.services.threat_service import create_threat_event
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AlertDispatcher:
|
||||
"""
|
||||
Routes scored threat events to storage, pub/sub, and structured logging.
|
||||
|
||||
MEDIUM+ severity events are persisted to PostgreSQL and published
|
||||
to the Redis alerts channel for WebSocket relay.
|
||||
All events are logged to stdout.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
redis_client: aioredis.Redis[str],
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
) -> None:
|
||||
self._redis = redis_client
|
||||
self._session_factory = session_factory
|
||||
|
||||
async def dispatch(self, scored: ScoredRequest) -> None:
|
||||
"""
|
||||
Handle a scored request from the pipeline's dispatch stage.
|
||||
"""
|
||||
logger.info(
|
||||
"threat_event severity=%s score=%.2f ip=%s path=%s rules=%s",
|
||||
scored.rule_result.severity,
|
||||
scored.rule_result.threat_score,
|
||||
scored.entry.ip,
|
||||
scored.entry.path,
|
||||
scored.rule_result.matched_rules,
|
||||
)
|
||||
|
||||
if scored.rule_result.severity in ("HIGH", "MEDIUM"):
|
||||
await self._store_event(scored)
|
||||
await self._publish_alert(scored)
|
||||
|
||||
async def _store_event(self, scored: ScoredRequest) -> None:
|
||||
"""
|
||||
Persist the scored request as a threat event in PostgreSQL.
|
||||
"""
|
||||
async with self._session_factory() as session:
|
||||
await create_threat_event(session, scored)
|
||||
await session.commit()
|
||||
|
||||
async def _publish_alert(self, scored: ScoredRequest) -> None:
|
||||
"""
|
||||
Publish a real-time alert to the Redis pub/sub channel.
|
||||
"""
|
||||
alert = WebSocketAlert(
|
||||
timestamp=scored.entry.timestamp,
|
||||
source_ip=scored.entry.ip,
|
||||
request_path=scored.entry.path,
|
||||
threat_score=scored.rule_result.threat_score,
|
||||
severity=scored.rule_result.severity,
|
||||
component_scores=scored.rule_result.component_scores,
|
||||
)
|
||||
await self._redis.publish(ALERTS_CHANNEL, alert.model_dump_json())
|
||||
|
|
@ -3,6 +3,7 @@
|
|||
rules.py
|
||||
"""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import NamedTuple
|
||||
|
||||
|
|
@ -20,17 +21,19 @@ from app.core.ingestion.parsers import ParsedLogEntry
|
|||
|
||||
class _PatternRule(NamedTuple):
|
||||
"""
|
||||
A regex-based detection rule applied to the request URI.
|
||||
A regex based detection rule applied to the request URI
|
||||
"""
|
||||
|
||||
name: str
|
||||
pattern: object
|
||||
pattern: re.Pattern[str]
|
||||
score: float
|
||||
|
||||
|
||||
class _ThresholdRule(NamedTuple):
|
||||
"""
|
||||
A threshold-based detection rule applied to a windowed feature.
|
||||
A threshold-based detection rule applied to a windowed feature
|
||||
"""
|
||||
|
||||
name: str
|
||||
feature_key: str
|
||||
threshold: float
|
||||
|
|
@ -60,6 +63,7 @@ class RuleResult:
|
|||
"""
|
||||
Output of the rule-based detection engine for a single request.
|
||||
"""
|
||||
|
||||
threat_score: float
|
||||
severity: str
|
||||
matched_rules: list[str] = field(default_factory=list)
|
||||
|
|
@ -81,7 +85,7 @@ class RuleEngine:
|
|||
"""
|
||||
Cold-start rule-based detection engine inspired by ModSecurity CRS.
|
||||
Scores requests using pattern matching, signature detection,
|
||||
and behavioral thresholds from windowed features.
|
||||
and behavioral thresholds from windowed features
|
||||
"""
|
||||
|
||||
def score_request(
|
||||
|
|
@ -109,10 +113,10 @@ class RuleEngine:
|
|||
if any(sig in ua_lower for sig in SCANNER_USER_AGENTS):
|
||||
matched.append(("SCANNER_UA", _SCANNER_UA_SCORE))
|
||||
|
||||
for rule in _THRESHOLD_RULES:
|
||||
value = features.get(rule.feature_key, 0)
|
||||
if isinstance(value, (int, float)) and value > rule.threshold:
|
||||
matched.append((rule.name, rule.score))
|
||||
for trule in _THRESHOLD_RULES:
|
||||
value = features.get(trule.feature_key, 0)
|
||||
if isinstance(value, (int, float)) and value > trule.threshold:
|
||||
matched.append((trule.name, trule.score))
|
||||
|
||||
if not matched:
|
||||
return RuleResult(threat_score=0.0, severity="LOW")
|
||||
|
|
@ -127,5 +131,5 @@ class RuleEngine:
|
|||
threat_score=threat_score,
|
||||
severity=_classify_severity(threat_score),
|
||||
matched_rules=[name for name, _ in matched],
|
||||
component_scores={name: score for name, score in matched},
|
||||
component_scores=dict(matched),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ class GeoResult:
|
|||
"""
|
||||
Structured GeoIP lookup result.
|
||||
"""
|
||||
|
||||
country: str | None
|
||||
city: str | None
|
||||
lat: float | None
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ class WindowAggregator:
|
|||
Per-IP sliding window feature aggregator backed by Redis sorted sets.
|
||||
"""
|
||||
|
||||
def __init__(self, redis_client: aioredis.Redis) -> None:
|
||||
def __init__(self, redis_client: aioredis.Redis[bytes]) -> None:
|
||||
self._redis = redis_client
|
||||
|
||||
async def record_and_aggregate(
|
||||
|
|
@ -66,17 +66,11 @@ class WindowAggregator:
|
|||
|
||||
pipe.zadd(keys["requests"], {request_id: timestamp})
|
||||
pipe.zadd(keys["paths"], {_hash_member(path): timestamp})
|
||||
pipe.zadd(
|
||||
keys["statuses"], {f"{status_code}:{request_id}": timestamp}
|
||||
)
|
||||
pipe.zadd(keys["statuses"], {f"{status_code}:{request_id}": timestamp})
|
||||
pipe.zadd(keys["uas"], {_hash_member(user_agent): timestamp})
|
||||
pipe.zadd(
|
||||
keys["sizes"], {f"{response_size}:{request_id}": timestamp}
|
||||
)
|
||||
pipe.zadd(keys["sizes"], {f"{response_size}:{request_id}": timestamp})
|
||||
pipe.zadd(keys["methods"], {f"{method}:{request_id}": timestamp})
|
||||
pipe.zadd(
|
||||
keys["depths"], {f"{path_depth}:{request_id}": timestamp}
|
||||
)
|
||||
pipe.zadd(keys["depths"], {f"{path_depth}:{request_id}": timestamp})
|
||||
|
||||
for key in keys.values():
|
||||
pipe.zremrangebyscore(key, "-inf", trim_boundary)
|
||||
|
|
@ -90,9 +84,7 @@ class WindowAggregator:
|
|||
pipe.zrangebyscore(keys["sizes"], w5m, "+inf")
|
||||
pipe.zrangebyscore(keys["methods"], w5m, "+inf")
|
||||
pipe.zrangebyscore(keys["depths"], w5m, "+inf")
|
||||
pipe.zrangebyscore(
|
||||
keys["requests"], w10m, "+inf", withscores=True
|
||||
)
|
||||
pipe.zrangebyscore(keys["requests"], w10m, "+inf", withscores=True)
|
||||
|
||||
for key in keys.values():
|
||||
pipe.expire(key, KEY_TTL)
|
||||
|
|
@ -135,9 +127,7 @@ def _error_rate(status_members: list[str]) -> float:
|
|||
"""
|
||||
if not status_members:
|
||||
return 0.0
|
||||
errors = sum(
|
||||
1 for m in status_members if int(m.split(":")[0]) >= 400
|
||||
)
|
||||
errors = sum(1 for m in status_members if int(m.split(":")[0]) >= 400)
|
||||
return errors / len(status_members)
|
||||
|
||||
|
||||
|
|
@ -160,9 +150,7 @@ def _method_entropy(method_members: list[str]) -> float:
|
|||
methods = [m.split(":")[0] for m in method_members]
|
||||
counts = Counter(methods)
|
||||
total = len(methods)
|
||||
return -sum(
|
||||
(c / total) * math.log2(c / total) for c in counts.values()
|
||||
)
|
||||
return -sum((c / total) * math.log2(c / total) for c in counts.values())
|
||||
|
||||
|
||||
def _status_diversity(status_members: list[str]) -> float:
|
||||
|
|
@ -195,10 +183,7 @@ def _inter_request_time_stats(
|
|||
if len(entries) < 2:
|
||||
return 0.0, 0.0
|
||||
timestamps = sorted(score for _, score in entries)
|
||||
deltas = [
|
||||
(timestamps[i + 1] - timestamps[i]) * 1000
|
||||
for i in range(len(timestamps) - 1)
|
||||
]
|
||||
deltas = [(timestamps[i + 1] - timestamps[i]) * 1000 for i in range(len(timestamps) - 1)]
|
||||
mean = sum(deltas) / len(deltas)
|
||||
if len(deltas) < 2:
|
||||
return mean, 0.0
|
||||
|
|
|
|||
|
|
@ -34,9 +34,9 @@ def encode_for_inference(
|
|||
if name in BOOLEAN_FEATURES:
|
||||
vector.append(1.0 if raw else 0.0)
|
||||
elif name in CATEGORICAL_ENCODERS:
|
||||
vector.append(float(CATEGORICAL_ENCODERS[name].get(raw, 0)))
|
||||
vector.append(float(CATEGORICAL_ENCODERS[name].get(str(raw), 0)))
|
||||
elif name == "country_code":
|
||||
vector.append(_encode_country(raw))
|
||||
vector.append(_encode_country(str(raw)))
|
||||
else:
|
||||
vector.append(float(raw))
|
||||
|
||||
|
|
|
|||
|
|
@ -25,9 +25,7 @@ def _shannon_entropy(s: str) -> float:
|
|||
return 0.0
|
||||
length = len(s)
|
||||
counts = Counter(s)
|
||||
return -sum(
|
||||
(c / length) * math.log2(c / length) for c in counts.values()
|
||||
)
|
||||
return -sum((c / length) * math.log2(c / length) for c in counts.values())
|
||||
|
||||
|
||||
def _is_private_ip(ip_str: str) -> bool:
|
||||
|
|
@ -63,9 +61,7 @@ def extract_request_features(
|
|||
"path_entropy": _shannon_entropy(entry.path),
|
||||
"path_length": path_len,
|
||||
"query_string_length": len(entry.query_string),
|
||||
"query_param_count": (
|
||||
len(entry.query_string.split("&")) if entry.query_string else 0
|
||||
),
|
||||
"query_param_count": (len(entry.query_string.split("&")) if entry.query_string else 0),
|
||||
"has_encoded_chars": bool(ENCODED_CHARS.search(full_uri)),
|
||||
"has_double_encoding": bool(DOUBLE_ENCODED.search(full_uri)),
|
||||
"status_code": entry.status_code,
|
||||
|
|
@ -77,9 +73,7 @@ def extract_request_features(
|
|||
"ua_length": len(entry.user_agent),
|
||||
"ua_entropy": _shannon_entropy(entry.user_agent),
|
||||
"is_known_bot": any(sig in ua_lower for sig in BOT_USER_AGENTS),
|
||||
"is_known_scanner": any(
|
||||
sig in ua_lower for sig in SCANNER_USER_AGENTS
|
||||
),
|
||||
"is_known_scanner": any(sig in ua_lower for sig in SCANNER_USER_AGENTS),
|
||||
"has_attack_pattern": bool(ATTACK_COMBINED.search(full_uri)),
|
||||
"special_char_ratio": non_alnum / path_len if path_len else 0.0,
|
||||
"file_extension": ext,
|
||||
|
|
|
|||
|
|
@ -93,12 +93,14 @@ CATEGORICAL_ENCODERS: dict[str, dict[str, int]] = {
|
|||
"file_extension": EXTENSION_MAP,
|
||||
}
|
||||
|
||||
BOOLEAN_FEATURES: frozenset[str] = frozenset({
|
||||
"has_encoded_chars",
|
||||
"has_double_encoding",
|
||||
"is_weekend",
|
||||
"is_known_bot",
|
||||
"is_known_scanner",
|
||||
"has_attack_pattern",
|
||||
"is_private_ip",
|
||||
})
|
||||
BOOLEAN_FEATURES: frozenset[str] = frozenset(
|
||||
{
|
||||
"has_encoded_chars",
|
||||
"has_double_encoding",
|
||||
"is_weekend",
|
||||
"is_known_bot",
|
||||
"is_known_scanner",
|
||||
"has_attack_pattern",
|
||||
"is_private_ip",
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,83 +3,87 @@
|
|||
signatures.py
|
||||
"""
|
||||
|
||||
BOT_USER_AGENTS: frozenset[str] = frozenset({
|
||||
"googlebot",
|
||||
"bingbot",
|
||||
"slurp",
|
||||
"duckduckbot",
|
||||
"baiduspider",
|
||||
"yandexbot",
|
||||
"sogou",
|
||||
"facebot",
|
||||
"ia_archiver",
|
||||
"applebot",
|
||||
"petalbot",
|
||||
"semrushbot",
|
||||
"ahrefsbot",
|
||||
"mj12bot",
|
||||
"dotbot",
|
||||
"rogerbot",
|
||||
"linkedinbot",
|
||||
"twitterbot",
|
||||
"gptbot",
|
||||
"claudebot",
|
||||
"amazonbot",
|
||||
"bytespider",
|
||||
"ccbot",
|
||||
"dataforseo",
|
||||
"seznambot",
|
||||
"megaindex",
|
||||
"blexbot",
|
||||
"exabot",
|
||||
"archive.org_bot",
|
||||
"mojeekbot",
|
||||
"uptimerobot",
|
||||
"deadlinkchecker",
|
||||
"sitebulb",
|
||||
"screaming frog",
|
||||
})
|
||||
BOT_USER_AGENTS: frozenset[str] = frozenset(
|
||||
{
|
||||
"googlebot",
|
||||
"bingbot",
|
||||
"slurp",
|
||||
"duckduckbot",
|
||||
"baiduspider",
|
||||
"yandexbot",
|
||||
"sogou",
|
||||
"facebot",
|
||||
"ia_archiver",
|
||||
"applebot",
|
||||
"petalbot",
|
||||
"semrushbot",
|
||||
"ahrefsbot",
|
||||
"mj12bot",
|
||||
"dotbot",
|
||||
"rogerbot",
|
||||
"linkedinbot",
|
||||
"twitterbot",
|
||||
"gptbot",
|
||||
"claudebot",
|
||||
"amazonbot",
|
||||
"bytespider",
|
||||
"ccbot",
|
||||
"dataforseo",
|
||||
"seznambot",
|
||||
"megaindex",
|
||||
"blexbot",
|
||||
"exabot",
|
||||
"archive.org_bot",
|
||||
"mojeekbot",
|
||||
"uptimerobot",
|
||||
"deadlinkchecker",
|
||||
"sitebulb",
|
||||
"screaming frog",
|
||||
}
|
||||
)
|
||||
|
||||
SCANNER_USER_AGENTS: frozenset[str] = frozenset({
|
||||
"nikto",
|
||||
"sqlmap",
|
||||
"nessus",
|
||||
"openvas",
|
||||
"acunetix",
|
||||
"w3af",
|
||||
"nmap",
|
||||
"masscan",
|
||||
"zgrab",
|
||||
"gobuster",
|
||||
"dirbuster",
|
||||
"dirb",
|
||||
"wfuzz",
|
||||
"ffuf",
|
||||
"nuclei",
|
||||
"burp",
|
||||
"zap",
|
||||
"arachni",
|
||||
"skipfish",
|
||||
"wpscan",
|
||||
"joomscan",
|
||||
"whatweb",
|
||||
"httprint",
|
||||
"fierce",
|
||||
"subfinder",
|
||||
"amass",
|
||||
"httpx",
|
||||
"jaeles",
|
||||
"xray",
|
||||
"gau",
|
||||
"hakrawler",
|
||||
"katana",
|
||||
"cariddi",
|
||||
"gospider",
|
||||
"feroxbuster",
|
||||
"rustbuster",
|
||||
"patator",
|
||||
"hydra",
|
||||
"medusa",
|
||||
"metasploit",
|
||||
"cobalt",
|
||||
})
|
||||
SCANNER_USER_AGENTS: frozenset[str] = frozenset(
|
||||
{
|
||||
"nikto",
|
||||
"sqlmap",
|
||||
"nessus",
|
||||
"openvas",
|
||||
"acunetix",
|
||||
"w3af",
|
||||
"nmap",
|
||||
"masscan",
|
||||
"zgrab",
|
||||
"gobuster",
|
||||
"dirbuster",
|
||||
"dirb",
|
||||
"wfuzz",
|
||||
"ffuf",
|
||||
"nuclei",
|
||||
"burp",
|
||||
"zap",
|
||||
"arachni",
|
||||
"skipfish",
|
||||
"wpscan",
|
||||
"joomscan",
|
||||
"whatweb",
|
||||
"httprint",
|
||||
"fierce",
|
||||
"subfinder",
|
||||
"amass",
|
||||
"httpx",
|
||||
"jaeles",
|
||||
"xray",
|
||||
"gau",
|
||||
"hakrawler",
|
||||
"katana",
|
||||
"cariddi",
|
||||
"gospider",
|
||||
"feroxbuster",
|
||||
"rustbuster",
|
||||
"patator",
|
||||
"hydra",
|
||||
"medusa",
|
||||
"metasploit",
|
||||
"cobalt",
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -72,9 +72,7 @@ def _parse_split(line: str) -> ParsedLogEntry | None:
|
|||
bracket_open = prefix.index("[")
|
||||
bracket_close = prefix.index("]")
|
||||
ip = prefix[:bracket_open].split()[0]
|
||||
timestamp = datetime.strptime(
|
||||
prefix[bracket_open + 1 : bracket_close], _TIMESTAMP_FMT
|
||||
)
|
||||
timestamp = datetime.strptime(prefix[bracket_open + 1 : bracket_close], _TIMESTAMP_FMT)
|
||||
|
||||
request_parts = request_line.split(" ", 2)
|
||||
method = request_parts[0]
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ class EnrichedRequest:
|
|||
"""
|
||||
A parsed log entry enriched with extracted features and GeoIP data.
|
||||
"""
|
||||
|
||||
entry: ParsedLogEntry
|
||||
features: dict[str, int | float | bool | str]
|
||||
feature_vector: list[float]
|
||||
|
|
@ -37,6 +38,7 @@ class ScoredRequest:
|
|||
"""
|
||||
A fully scored request ready for dispatch.
|
||||
"""
|
||||
|
||||
entry: ParsedLogEntry
|
||||
features: dict[str, int | float | bool | str]
|
||||
feature_vector: list[float]
|
||||
|
|
@ -56,7 +58,7 @@ class Pipeline:
|
|||
|
||||
def __init__(
|
||||
self,
|
||||
redis_client: aioredis.Redis,
|
||||
redis_client: aioredis.Redis[bytes],
|
||||
rule_engine: RuleEngine,
|
||||
geoip: GeoIPService | None = None,
|
||||
on_result: Callable[[ScoredRequest], Awaitable[None]] | None = None,
|
||||
|
|
@ -82,7 +84,7 @@ class Pipeline:
|
|||
self._rule_engine = rule_engine
|
||||
self._geoip = geoip
|
||||
self._on_result = on_result
|
||||
self._tasks: list[asyncio.Task] = []
|
||||
self._tasks: list[asyncio.Task[None]] = []
|
||||
|
||||
async def start(self) -> None:
|
||||
"""
|
||||
|
|
@ -147,7 +149,7 @@ class Pipeline:
|
|||
ip=entry.ip,
|
||||
request_id=uuid.uuid4().hex,
|
||||
path=entry.path,
|
||||
path_depth=per_request["path_depth"],
|
||||
path_depth=int(per_request["path_depth"]),
|
||||
method=entry.method,
|
||||
status_code=entry.status_code,
|
||||
user_agent=entry.user_agent,
|
||||
|
|
@ -182,7 +184,8 @@ class Pipeline:
|
|||
break
|
||||
try:
|
||||
rule_result = self._rule_engine.score_request(
|
||||
enriched.features, enriched.entry,
|
||||
enriched.features,
|
||||
enriched.entry,
|
||||
)
|
||||
await self._alert_queue.put(
|
||||
ScoredRequest(
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ tailer.py
|
|||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from io import TextIOWrapper
|
||||
from pathlib import Path
|
||||
|
||||
from watchdog.events import (
|
||||
|
|
@ -28,14 +29,14 @@ class _LogHandler(FileSystemEventHandler):
|
|||
def __init__(
|
||||
self,
|
||||
target: str,
|
||||
queue: asyncio.Queue[str],
|
||||
queue: asyncio.Queue[str | None],
|
||||
loop: asyncio.AbstractEventLoop,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._target = target
|
||||
self._queue = queue
|
||||
self._loop = loop
|
||||
self._file = None
|
||||
self._file: TextIOWrapper | None = None
|
||||
self._inode: int | None = None
|
||||
self._open_target()
|
||||
|
||||
|
|
@ -44,7 +45,7 @@ class _LogHandler(FileSystemEventHandler):
|
|||
Open the target log file and seek to the end.
|
||||
"""
|
||||
try:
|
||||
self._file = open(self._target, encoding="utf-8", errors="replace")
|
||||
self._file = open(self._target, encoding="utf-8", errors="replace") # noqa: SIM115
|
||||
self._file.seek(0, os.SEEK_END)
|
||||
self._inode = os.stat(self._target).st_ino
|
||||
logger.info("Tailing %s (inode %s)", self._target, self._inode)
|
||||
|
|
@ -75,7 +76,7 @@ class _LogHandler(FileSystemEventHandler):
|
|||
self._file.close()
|
||||
|
||||
try:
|
||||
self._file = open(self._target, encoding="utf-8", errors="replace")
|
||||
self._file = open(self._target, encoding="utf-8", errors="replace") # noqa: SIM115
|
||||
self._inode = os.stat(self._target).st_ino
|
||||
logger.info("Rotated to new %s (inode %s)", self._target, self._inode)
|
||||
except FileNotFoundError:
|
||||
|
|
@ -92,14 +93,14 @@ class _LogHandler(FileSystemEventHandler):
|
|||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
def on_modified(self, event: FileModifiedEvent) -> None:
|
||||
def on_modified(self, event: FileModifiedEvent) -> None: # type: ignore[override]
|
||||
"""
|
||||
Handle new data appended to the log file.
|
||||
"""
|
||||
if not isinstance(event, FileModifiedEvent) or event.is_directory:
|
||||
return
|
||||
|
||||
if Path(event.src_path).resolve() != Path(self._target).resolve():
|
||||
if Path(str(event.src_path)).resolve() != Path(self._target).resolve():
|
||||
return
|
||||
|
||||
if self._inode_changed():
|
||||
|
|
@ -108,25 +109,25 @@ class _LogHandler(FileSystemEventHandler):
|
|||
|
||||
self._read_new_lines()
|
||||
|
||||
def on_moved(self, event: FileMovedEvent) -> None:
|
||||
def on_moved(self, event: FileMovedEvent) -> None: # type: ignore[override]
|
||||
"""
|
||||
Handle log rotation via rename (access.log -> access.log.1).
|
||||
"""
|
||||
if not isinstance(event, FileMovedEvent):
|
||||
return
|
||||
|
||||
if Path(event.src_path).resolve() == Path(self._target).resolve():
|
||||
if Path(str(event.src_path)).resolve() == Path(self._target).resolve():
|
||||
logger.info("Log rotated: %s -> %s", event.src_path, event.dest_path)
|
||||
self._handle_rotation()
|
||||
|
||||
def on_created(self, event: FileCreatedEvent) -> None:
|
||||
def on_created(self, event: FileCreatedEvent) -> None: # type: ignore[override]
|
||||
"""
|
||||
Handle log rotation where a new file is created at the target path.
|
||||
"""
|
||||
if not isinstance(event, FileCreatedEvent) or event.is_directory:
|
||||
return
|
||||
|
||||
if Path(event.src_path).resolve() == Path(self._target).resolve():
|
||||
if Path(str(event.src_path)).resolve() == Path(self._target).resolve():
|
||||
logger.info("New log file created: %s", event.src_path)
|
||||
self._handle_rotation()
|
||||
|
||||
|
|
@ -148,7 +149,7 @@ class LogTailer:
|
|||
def __init__(
|
||||
self,
|
||||
log_path: str,
|
||||
queue: asyncio.Queue[str],
|
||||
queue: asyncio.Queue[str | None],
|
||||
loop: asyncio.AbstractEventLoop,
|
||||
) -> None:
|
||||
self._log_path = log_path
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ class RedisManager:
|
|||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._client: aioredis.Redis | None = None
|
||||
self._client: aioredis.Redis[str] | None = None
|
||||
|
||||
async def connect(self) -> None:
|
||||
"""
|
||||
|
|
@ -30,11 +30,11 @@ class RedisManager:
|
|||
Close the Redis connection and release resources.
|
||||
"""
|
||||
if self._client:
|
||||
await self._client.aclose()
|
||||
await self._client.aclose() # type: ignore[attr-defined]
|
||||
self._client = None
|
||||
|
||||
@property
|
||||
def client(self) -> aioredis.Redis | None:
|
||||
def client(self) -> aioredis.Redis[str] | None:
|
||||
"""
|
||||
Return the active Redis client or None if not connected.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -3,34 +3,106 @@
|
|||
factory.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlmodel import SQLModel
|
||||
|
||||
from app.config import settings
|
||||
from app.core.alerts.dispatcher import AlertDispatcher
|
||||
from app.core.detection.rules import RuleEngine
|
||||
from app.core.enrichment.geoip import GeoIPService
|
||||
from app.core.ingestion.pipeline import Pipeline
|
||||
from app.core.ingestion.tailer import LogTailer
|
||||
from app.core.redis_manager import redis_manager
|
||||
from app.models import model_metadata as _model_metadata_reg # noqa: F401
|
||||
from app.models import threat_event as _threat_event_reg # noqa: F401
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
"""
|
||||
Manage application startup and shutdown lifecycle
|
||||
Manage application startup and shutdown lifecycle.
|
||||
"""
|
||||
app.state.startup_time = time.monotonic()
|
||||
app.state.pipeline_running = False
|
||||
|
||||
engine = create_async_engine(settings.database_url)
|
||||
app.state.db_engine = engine
|
||||
app.state.session_factory = async_sessionmaker(
|
||||
engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False,
|
||||
)
|
||||
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(SQLModel.metadata.create_all)
|
||||
logger.info("Database tables verified")
|
||||
|
||||
await redis_manager.connect()
|
||||
|
||||
geoip = GeoIPService(settings.geoip_db_path)
|
||||
|
||||
redis_client = redis_manager.client
|
||||
assert redis_client is not None
|
||||
|
||||
dispatcher = AlertDispatcher(
|
||||
redis_client=redis_client,
|
||||
session_factory=app.state.session_factory,
|
||||
)
|
||||
|
||||
pipeline = Pipeline(
|
||||
redis_client=redis_client, # type: ignore[arg-type]
|
||||
rule_engine=RuleEngine(),
|
||||
geoip=geoip,
|
||||
on_result=dispatcher.dispatch,
|
||||
raw_queue_size=settings.raw_queue_size,
|
||||
parsed_queue_size=settings.parsed_queue_size,
|
||||
feature_queue_size=settings.feature_queue_size,
|
||||
alert_queue_size=settings.alert_queue_size,
|
||||
)
|
||||
await pipeline.start()
|
||||
|
||||
tailer = None
|
||||
log_dir = Path(settings.nginx_log_path).resolve().parent
|
||||
if log_dir.is_dir():
|
||||
loop = asyncio.get_running_loop()
|
||||
tailer = LogTailer(settings.nginx_log_path, pipeline.raw_queue, loop)
|
||||
tailer.start()
|
||||
else:
|
||||
logger.warning("Log directory %s not found — tailer disabled", log_dir)
|
||||
|
||||
app.state.pipeline = pipeline
|
||||
app.state.tailer = tailer
|
||||
app.state.geoip = geoip
|
||||
app.state.pipeline_running = True
|
||||
|
||||
logger.info("AngelusVigil started — pipeline active")
|
||||
|
||||
yield
|
||||
|
||||
app.state.pipeline_running = False
|
||||
if tailer is not None:
|
||||
tailer.stop()
|
||||
await pipeline.stop()
|
||||
geoip.close()
|
||||
await redis_manager.disconnect()
|
||||
await engine.dispose()
|
||||
|
||||
logger.info("AngelusVigil shut down cleanly")
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
"""
|
||||
Build and configure the AngelusVigil FastAPI application
|
||||
Build and configure the AngelusVigil FastAPI application.
|
||||
"""
|
||||
app = FastAPI(
|
||||
title=settings.app_name,
|
||||
|
|
@ -42,8 +114,15 @@ def create_app() -> FastAPI:
|
|||
app.state.pipeline_running = False
|
||||
|
||||
from app.api.health import router as health_router
|
||||
from app.api.models_api import router as models_router
|
||||
from app.api.stats import router as stats_router
|
||||
from app.api.threats import router as threats_router
|
||||
from app.api.websocket import router as ws_router
|
||||
|
||||
app.include_router(health_router)
|
||||
app.include_router(threats_router)
|
||||
app.include_router(stats_router)
|
||||
app.include_router(models_router)
|
||||
app.include_router(ws_router)
|
||||
|
||||
return app
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ class TimestampedModel(SQLModel):
|
|||
"""
|
||||
|
||||
id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True)
|
||||
created_at: datetime = Field(
|
||||
created_at: datetime = Field( # type: ignore[call-overload]
|
||||
sa_type=DateTime(timezone=True),
|
||||
sa_column_kwargs={"server_default": text("CURRENT_TIMESTAMP")},
|
||||
nullable=False,
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ class ModelMetadata(TimestampedModel, table=True):
|
|||
model_type: str = Field(max_length=30)
|
||||
version: str = Field(max_length=64)
|
||||
training_samples: int
|
||||
metrics: dict = Field(sa_column=Column(JSON, nullable=False))
|
||||
metrics: dict[str, object] = Field(sa_column=Column(JSON, nullable=False))
|
||||
artifact_path: str
|
||||
is_active: bool = Field(default=False)
|
||||
mlflow_run_id: str | None = Field(default=None, max_length=64)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,14 @@
|
|||
threat_event.py
|
||||
"""
|
||||
|
||||
from sqlalchemy import Column, Float, Index, JSON, SmallInteger, text
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
Float,
|
||||
Index,
|
||||
JSON,
|
||||
SmallInteger,
|
||||
text,
|
||||
)
|
||||
from sqlmodel import Field
|
||||
|
||||
from app.models.base import TimestampedModel
|
||||
|
|
@ -30,25 +37,17 @@ class ThreatEvent(TimestampedModel, table=True):
|
|||
source_ip: str = Field(max_length=45)
|
||||
request_method: str = Field(max_length=10)
|
||||
request_path: str
|
||||
status_code: int = Field(
|
||||
sa_column=Column(SmallInteger, nullable=False)
|
||||
)
|
||||
status_code: int = Field(sa_column=Column(SmallInteger, nullable=False))
|
||||
response_size: int
|
||||
user_agent: str
|
||||
threat_score: float = Field(
|
||||
sa_column=Column(Float, nullable=False)
|
||||
)
|
||||
threat_score: float = Field(sa_column=Column(Float, nullable=False))
|
||||
severity: str = Field(max_length=6)
|
||||
component_scores: dict[str, float] = Field(
|
||||
sa_column=Column(JSON, nullable=False)
|
||||
)
|
||||
component_scores: dict[str, float] = Field(sa_column=Column(JSON, nullable=False))
|
||||
geo_country: str | None = Field(default=None, max_length=2)
|
||||
geo_city: str | None = Field(default=None, max_length=255)
|
||||
geo_lat: float | None = Field(default=None)
|
||||
geo_lon: float | None = Field(default=None)
|
||||
feature_vector: list[float] = Field(
|
||||
sa_column=Column(JSON, nullable=False)
|
||||
)
|
||||
feature_vector: list[float] = Field(sa_column=Column(JSON, nullable=False))
|
||||
matched_rules: list[str] | None = Field(
|
||||
default=None, sa_column=Column(JSON, nullable=True)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
stats.py
|
||||
"""
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class SeverityBreakdown(BaseModel):
|
||||
"""
|
||||
Count of threats per severity tier.
|
||||
"""
|
||||
|
||||
high: int = 0
|
||||
medium: int = 0
|
||||
low: int = 0
|
||||
|
||||
|
||||
class IPStatEntry(BaseModel):
|
||||
"""
|
||||
Source IP with associated threat count.
|
||||
"""
|
||||
|
||||
source_ip: str
|
||||
count: int
|
||||
|
||||
|
||||
class PathStatEntry(BaseModel):
|
||||
"""
|
||||
Request path with associated threat count.
|
||||
"""
|
||||
|
||||
path: str
|
||||
count: int
|
||||
|
||||
|
||||
class StatsResponse(BaseModel):
|
||||
"""
|
||||
Aggregate threat statistics for a given time range.
|
||||
"""
|
||||
|
||||
time_range: str
|
||||
total_requests: int
|
||||
threats_detected: int
|
||||
severity_breakdown: SeverityBreakdown
|
||||
top_source_ips: list[IPStatEntry]
|
||||
top_attacked_paths: list[PathStatEntry]
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
threats.py
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class GeoInfo(BaseModel):
|
||||
"""
|
||||
Geographic location data from GeoIP lookup.
|
||||
"""
|
||||
|
||||
country: str | None = None
|
||||
city: str | None = None
|
||||
lat: float | None = None
|
||||
lon: float | None = None
|
||||
|
||||
|
||||
class ThreatEventResponse(BaseModel):
|
||||
"""
|
||||
Single threat event returned by the API.
|
||||
"""
|
||||
|
||||
id: uuid.UUID
|
||||
created_at: datetime
|
||||
source_ip: str
|
||||
request_method: str
|
||||
request_path: str
|
||||
status_code: int
|
||||
response_size: int
|
||||
user_agent: str
|
||||
threat_score: float
|
||||
severity: Literal["HIGH", "MEDIUM", "LOW"]
|
||||
component_scores: dict[str, float]
|
||||
geo: GeoInfo
|
||||
matched_rules: list[str] | None = None
|
||||
model_version: str | None = None
|
||||
reviewed: bool = False
|
||||
review_label: str | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class ThreatListResponse(BaseModel):
|
||||
"""
|
||||
Paginated list of threat events.
|
||||
"""
|
||||
|
||||
total: int
|
||||
limit: int
|
||||
offset: int
|
||||
items: list[ThreatEventResponse]
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
websocket.py
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class WebSocketAlert(BaseModel):
|
||||
"""
|
||||
Real-time threat alert broadcast over WebSocket.
|
||||
"""
|
||||
|
||||
event: Literal["threat"] = "threat"
|
||||
timestamp: datetime
|
||||
source_ip: str
|
||||
request_path: str
|
||||
threat_score: float
|
||||
severity: str
|
||||
component_scores: dict[str, float]
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
__init__.py
|
||||
"""
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
stats_service.py
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, UTC
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.threat_event import ThreatEvent
|
||||
from app.schemas.stats import (
|
||||
IPStatEntry,
|
||||
PathStatEntry,
|
||||
SeverityBreakdown,
|
||||
StatsResponse,
|
||||
)
|
||||
|
||||
_RANGE_MAP: dict[str, timedelta] = {
|
||||
"1h": timedelta(hours=1),
|
||||
"6h": timedelta(hours=6),
|
||||
"24h": timedelta(hours=24),
|
||||
"7d": timedelta(days=7),
|
||||
"30d": timedelta(days=30),
|
||||
}
|
||||
|
||||
|
||||
async def get_stats(
|
||||
session: AsyncSession,
|
||||
time_range: str = "24h",
|
||||
) -> StatsResponse:
|
||||
"""
|
||||
Compute aggregate threat statistics for a given time window.
|
||||
"""
|
||||
delta = _RANGE_MAP.get(time_range, timedelta(hours=24))
|
||||
cutoff = datetime.now(UTC) - delta
|
||||
|
||||
base = select(ThreatEvent).where(ThreatEvent.created_at >= cutoff) # type: ignore[arg-type]
|
||||
|
||||
total_q = select(func.count()).select_from(base.subquery())
|
||||
total = (await session.execute(total_q)).scalar_one()
|
||||
|
||||
sev_q = (
|
||||
select(ThreatEvent.severity, func.count()) # type: ignore[call-overload]
|
||||
.where(ThreatEvent.created_at >= cutoff)
|
||||
.group_by(ThreatEvent.severity)
|
||||
)
|
||||
sev_rows = (await session.execute(sev_q)).all()
|
||||
sev_map = {row[0]: row[1] for row in sev_rows}
|
||||
|
||||
threats_detected = sev_map.get("HIGH", 0) + sev_map.get("MEDIUM", 0)
|
||||
|
||||
ip_q = (
|
||||
select(ThreatEvent.source_ip, func.count().label("cnt")) # type: ignore[call-overload]
|
||||
.where(ThreatEvent.created_at >= cutoff)
|
||||
.group_by(ThreatEvent.source_ip)
|
||||
.order_by(func.count().desc())
|
||||
.limit(10)
|
||||
)
|
||||
ip_rows = (await session.execute(ip_q)).all()
|
||||
|
||||
path_q = (
|
||||
select(ThreatEvent.request_path, func.count().label("cnt")) # type: ignore[call-overload]
|
||||
.where(ThreatEvent.created_at >= cutoff)
|
||||
.group_by(ThreatEvent.request_path)
|
||||
.order_by(func.count().desc())
|
||||
.limit(10)
|
||||
)
|
||||
path_rows = (await session.execute(path_q)).all()
|
||||
|
||||
return StatsResponse(
|
||||
time_range=time_range,
|
||||
total_requests=total,
|
||||
threats_detected=threats_detected,
|
||||
severity_breakdown=SeverityBreakdown(
|
||||
high=sev_map.get("HIGH", 0),
|
||||
medium=sev_map.get("MEDIUM", 0),
|
||||
low=sev_map.get("LOW", 0),
|
||||
),
|
||||
top_source_ips=[IPStatEntry(source_ip=row[0], count=row[1]) for row in ip_rows],
|
||||
top_attacked_paths=[PathStatEntry(path=row[0], count=row[1]) for row in path_rows],
|
||||
)
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
threat_service.py
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.ingestion.pipeline import ScoredRequest
|
||||
from app.models.threat_event import ThreatEvent
|
||||
from app.schemas.threats import (
|
||||
GeoInfo,
|
||||
ThreatEventResponse,
|
||||
ThreatListResponse,
|
||||
)
|
||||
|
||||
|
||||
def _to_response(event: ThreatEvent) -> ThreatEventResponse:
|
||||
"""
|
||||
Convert a ThreatEvent DB model to an API response schema.
|
||||
"""
|
||||
return ThreatEventResponse(
|
||||
id=event.id,
|
||||
created_at=event.created_at,
|
||||
source_ip=event.source_ip,
|
||||
request_method=event.request_method,
|
||||
request_path=event.request_path,
|
||||
status_code=event.status_code,
|
||||
response_size=event.response_size,
|
||||
user_agent=event.user_agent,
|
||||
threat_score=event.threat_score,
|
||||
severity=event.severity, # type: ignore[arg-type]
|
||||
component_scores=event.component_scores,
|
||||
geo=GeoInfo(
|
||||
country=event.geo_country,
|
||||
city=event.geo_city,
|
||||
lat=event.geo_lat,
|
||||
lon=event.geo_lon,
|
||||
),
|
||||
matched_rules=event.matched_rules,
|
||||
model_version=event.model_version,
|
||||
reviewed=event.reviewed,
|
||||
review_label=event.review_label,
|
||||
)
|
||||
|
||||
|
||||
async def get_threats(
|
||||
session: AsyncSession,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
severity: str | None = None,
|
||||
source_ip: str | None = None,
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
) -> ThreatListResponse:
|
||||
"""
|
||||
Query threat events with optional filters, returning a paginated response.
|
||||
"""
|
||||
query = select(ThreatEvent)
|
||||
count_query = select(func.count()).select_from(ThreatEvent)
|
||||
|
||||
if severity:
|
||||
query = query.where(ThreatEvent.severity == severity.upper()) # type: ignore[arg-type]
|
||||
count_query = count_query.where(ThreatEvent.severity == severity.upper()) # type: ignore[arg-type]
|
||||
if source_ip:
|
||||
query = query.where(ThreatEvent.source_ip == source_ip) # type: ignore[arg-type]
|
||||
count_query = count_query.where(ThreatEvent.source_ip == source_ip) # type: ignore[arg-type]
|
||||
if since:
|
||||
query = query.where(ThreatEvent.created_at >= since) # type: ignore[arg-type]
|
||||
count_query = count_query.where(ThreatEvent.created_at >= since) # type: ignore[arg-type]
|
||||
if until:
|
||||
query = query.where(ThreatEvent.created_at <= until) # type: ignore[arg-type]
|
||||
count_query = count_query.where(ThreatEvent.created_at <= until) # type: ignore[arg-type]
|
||||
|
||||
query = query.order_by(ThreatEvent.created_at.desc()) # type: ignore[attr-defined]
|
||||
query = query.offset(offset).limit(limit)
|
||||
|
||||
total = (await session.execute(count_query)).scalar_one()
|
||||
rows = (await session.execute(query)).scalars().all()
|
||||
|
||||
return ThreatListResponse(
|
||||
total=total,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
items=[_to_response(row) for row in rows],
|
||||
)
|
||||
|
||||
|
||||
async def get_threat_by_id(
|
||||
session: AsyncSession,
|
||||
threat_id: uuid.UUID,
|
||||
) -> ThreatEventResponse | None:
|
||||
"""
|
||||
Fetch a single threat event by its UUID.
|
||||
"""
|
||||
result = await session.get(ThreatEvent, threat_id)
|
||||
if result is None:
|
||||
return None
|
||||
return _to_response(result)
|
||||
|
||||
|
||||
async def create_threat_event(
|
||||
session: AsyncSession,
|
||||
scored: ScoredRequest,
|
||||
) -> ThreatEvent:
|
||||
"""
|
||||
Persist a scored request as a threat event in the database.
|
||||
"""
|
||||
event = ThreatEvent(
|
||||
source_ip=scored.entry.ip,
|
||||
request_method=scored.entry.method,
|
||||
request_path=scored.entry.path,
|
||||
status_code=scored.entry.status_code,
|
||||
response_size=scored.entry.response_size,
|
||||
user_agent=scored.entry.user_agent,
|
||||
threat_score=scored.rule_result.threat_score,
|
||||
severity=scored.rule_result.severity,
|
||||
component_scores=scored.rule_result.component_scores,
|
||||
geo_country=scored.geo.country if scored.geo else None,
|
||||
geo_city=scored.geo.city if scored.geo else None,
|
||||
geo_lat=scored.geo.lat if scored.geo else None,
|
||||
geo_lon=scored.geo.lon if scored.geo else None,
|
||||
feature_vector=scored.feature_vector,
|
||||
matched_rules=scored.rule_result.matched_rules or None,
|
||||
model_version="rules-v1",
|
||||
)
|
||||
session.add(event)
|
||||
await session.flush()
|
||||
return event
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
main.py
|
||||
"""
|
||||
|
||||
import typer
|
||||
|
||||
app = typer.Typer(
|
||||
name="vigil",
|
||||
help="AngelusVigil — AI-powered threat detection engine",
|
||||
no_args_is_help=True,
|
||||
)
|
||||
|
||||
|
||||
@app.command()
|
||||
def serve(
|
||||
host: str = typer.Option("0.0.0.0", help="Bind address"),
|
||||
port: int = typer.Option(8000, help="Bind port"),
|
||||
reload: bool = typer.Option(False, help="Enable auto-reload for development"),
|
||||
) -> None:
|
||||
"""
|
||||
Start the AngelusVigil API server.
|
||||
"""
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(
|
||||
"app.main:app",
|
||||
host=host,
|
||||
port=port,
|
||||
reload=reload,
|
||||
)
|
||||
|
||||
|
||||
@app.command()
|
||||
def config() -> None:
|
||||
"""
|
||||
Print the current configuration (secrets redacted).
|
||||
"""
|
||||
from app.config import settings
|
||||
|
||||
safe_fields = {}
|
||||
for key, value in settings.model_dump().items():
|
||||
if any(secret in key for secret in ("key", "password", "secret", "token")):
|
||||
safe_fields[key] = "***REDACTED***"
|
||||
elif "url" in key and "@" in str(value):
|
||||
safe_fields[key] = _redact_url(str(value))
|
||||
else:
|
||||
safe_fields[key] = value
|
||||
|
||||
for key, value in sorted(safe_fields.items()):
|
||||
typer.echo(f" {key}: {value}")
|
||||
|
||||
|
||||
@app.command()
|
||||
def health(
|
||||
url: str = typer.Option(
|
||||
"http://localhost:8000",
|
||||
help="Base URL of the running server",
|
||||
),
|
||||
) -> None:
|
||||
"""
|
||||
Ping the running server's /health endpoint.
|
||||
"""
|
||||
import httpx
|
||||
|
||||
try:
|
||||
response = httpx.get(f"{url}/health", timeout=5.0)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
typer.echo(f" status: {data.get('status', 'unknown')}")
|
||||
typer.echo(f" uptime: {data.get('uptime_seconds', 0):.0f}s")
|
||||
typer.echo(f" pipeline: {'running' if data.get('pipeline_running') else 'stopped'}")
|
||||
except httpx.ConnectError:
|
||||
typer.echo("Error: cannot connect to server", err=True)
|
||||
raise typer.Exit(code=1) from None
|
||||
except httpx.HTTPStatusError as exc:
|
||||
typer.echo(f"Error: server returned {exc.response.status_code}", err=True)
|
||||
raise typer.Exit(code=1) from None
|
||||
|
||||
|
||||
def _redact_url(url: str) -> str:
|
||||
"""
|
||||
Replace the user:password portion of a database URL with ***:***.
|
||||
"""
|
||||
if "://" not in url or "@" not in url:
|
||||
return url
|
||||
scheme, rest = url.split("://", 1)
|
||||
_, host_part = rest.rsplit("@", 1)
|
||||
return f"{scheme}://***:***@{host_part}"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
|
|
@ -157,7 +157,6 @@ load-plugins = [
|
|||
"pylint_pydantic",
|
||||
]
|
||||
persistent = true
|
||||
suggestion-mode = true
|
||||
ignore = [
|
||||
"venv",
|
||||
".venv",
|
||||
|
|
@ -198,20 +197,16 @@ disable = [
|
|||
"W0718",
|
||||
"E0611",
|
||||
"E1101",
|
||||
"C0415",
|
||||
"E1102",
|
||||
"R1732",
|
||||
]
|
||||
|
||||
[tool.pylint.design]
|
||||
max-args = 7
|
||||
max-args = 12
|
||||
max-attributes = 10
|
||||
max-locals = 30
|
||||
max-positional-arguments = 7
|
||||
|
||||
[tool.pylint."messages control"]
|
||||
per-file-ignores = [
|
||||
"alembic/env.py:W0611",
|
||||
"app/services/*:C0121",
|
||||
"tests/*:W0212,W0621",
|
||||
]
|
||||
max-locals = 35
|
||||
max-positional-arguments = 12
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
asyncio_mode = "auto"
|
||||
|
|
|
|||
|
|
@ -3,9 +3,17 @@
|
|||
conftest.py
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
from sqlmodel import SQLModel
|
||||
|
||||
from app.api.deps import get_session
|
||||
from app.config import Settings
|
||||
from app.factory import create_app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -16,8 +24,65 @@ def test_settings() -> Settings:
|
|||
return Settings(
|
||||
env="testing",
|
||||
debug=False,
|
||||
database_url="sqlite+aiosqlite:///test.db",
|
||||
database_url="sqlite+aiosqlite://",
|
||||
redis_url="redis://localhost:6379/1",
|
||||
nginx_log_path="/tmp/test-access.log",
|
||||
geoip_db_path="/tmp/nonexistent.mmdb",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def db_engine():
|
||||
"""
|
||||
In-memory SQLite engine shared across all sessions via StaticPool.
|
||||
"""
|
||||
from app.models.threat_event import ThreatEvent # noqa: F401
|
||||
|
||||
engine = create_async_engine(
|
||||
"sqlite+aiosqlite://",
|
||||
connect_args={"check_same_thread": False},
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(SQLModel.metadata.create_all)
|
||||
yield engine
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def db_session(db_engine) -> AsyncIterator[AsyncSession]:
|
||||
"""
|
||||
Async session bound to the shared in-memory DB.
|
||||
"""
|
||||
factory = async_sessionmaker(
|
||||
db_engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False,
|
||||
)
|
||||
async with factory() as session:
|
||||
yield session
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def db_client(db_engine) -> AsyncIterator[AsyncClient]:
|
||||
"""
|
||||
HTTPX async client with DB dependency overridden to use in-memory SQLite.
|
||||
"""
|
||||
factory = async_sessionmaker(
|
||||
db_engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False,
|
||||
)
|
||||
|
||||
test_app = create_app()
|
||||
|
||||
async def override_get_session() -> AsyncIterator[AsyncSession]:
|
||||
async with factory() as session:
|
||||
yield session
|
||||
await session.commit()
|
||||
|
||||
test_app.dependency_overrides[get_session] = override_get_session
|
||||
|
||||
transport = ASGITransport(app=test_app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
yield client
|
||||
|
|
|
|||
|
|
@ -3,10 +3,14 @@
|
|||
test_api.py
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, UTC
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.main import app
|
||||
from app.models.threat_event import ThreatEvent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -55,3 +59,147 @@ async def test_ready_returns_check_structure() -> None:
|
|||
assert "database" in data["checks"]
|
||||
assert "redis" in data["checks"]
|
||||
assert "models_loaded" in data["checks"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_threats_empty(db_client) -> None:
|
||||
"""
|
||||
GET /threats on an empty database returns zero items.
|
||||
"""
|
||||
response = await db_client.get("/threats")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["total"] == 0
|
||||
assert data["limit"] == 50
|
||||
assert data["offset"] == 0
|
||||
assert data["items"] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_threat_not_found(db_client) -> None:
|
||||
"""
|
||||
GET /threats/{random_id} returns 404 when the event does not exist.
|
||||
"""
|
||||
fake_id = uuid.uuid4()
|
||||
response = await db_client.get(f"/threats/{fake_id}")
|
||||
|
||||
assert response.status_code == 404
|
||||
assert response.json()["detail"] == "Threat event not found"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_threat_by_id(db_session, db_client) -> None:
|
||||
"""
|
||||
Seed a threat event, then fetch it by ID.
|
||||
"""
|
||||
event_id = uuid.uuid4()
|
||||
event = ThreatEvent(
|
||||
id=event_id,
|
||||
created_at=datetime.now(UTC),
|
||||
source_ip="10.0.0.1",
|
||||
request_method="GET",
|
||||
request_path="/admin",
|
||||
status_code=200,
|
||||
response_size=512,
|
||||
user_agent="TestBot/1.0",
|
||||
threat_score=0.85,
|
||||
severity="HIGH",
|
||||
component_scores={"SQL_INJECTION": 0.85},
|
||||
feature_vector=[0.0] * 35,
|
||||
matched_rules=["SQL_INJECTION"],
|
||||
)
|
||||
db_session.add(event)
|
||||
await db_session.commit()
|
||||
|
||||
response = await db_client.get(f"/threats/{event_id}")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["id"] == str(event_id)
|
||||
assert data["source_ip"] == "10.0.0.1"
|
||||
assert data["threat_score"] == 0.85
|
||||
assert data["severity"] == "HIGH"
|
||||
assert data["matched_rules"] == ["SQL_INJECTION"]
|
||||
assert data["geo"]["country"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_threats_severity_filter(db_session, db_client) -> None:
|
||||
"""
|
||||
Seed threats with different severities and filter by HIGH.
|
||||
"""
|
||||
now = datetime.now(UTC)
|
||||
for severity, score in [("HIGH", 0.9), ("MEDIUM", 0.6), ("LOW", 0.3)]:
|
||||
event = ThreatEvent(
|
||||
id=uuid.uuid4(),
|
||||
created_at=now,
|
||||
source_ip="192.168.1.1",
|
||||
request_method="GET",
|
||||
request_path="/test",
|
||||
status_code=200,
|
||||
response_size=100,
|
||||
user_agent="Mozilla/5.0",
|
||||
threat_score=score,
|
||||
severity=severity,
|
||||
component_scores={},
|
||||
feature_vector=[0.0] * 35,
|
||||
)
|
||||
db_session.add(event)
|
||||
await db_session.commit()
|
||||
|
||||
response = await db_client.get("/threats", params={"severity": "HIGH"})
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["total"] == 1
|
||||
assert data["items"][0]["severity"] == "HIGH"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stats_empty_window(db_client) -> None:
|
||||
"""
|
||||
GET /stats on an empty database returns zero counts.
|
||||
"""
|
||||
response = await db_client.get("/stats")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["time_range"] == "24h"
|
||||
assert data["total_requests"] == 0
|
||||
assert data["threats_detected"] == 0
|
||||
assert data["severity_breakdown"]["high"] == 0
|
||||
assert data["severity_breakdown"]["medium"] == 0
|
||||
assert data["severity_breakdown"]["low"] == 0
|
||||
assert data["top_source_ips"] == []
|
||||
assert data["top_attacked_paths"] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_status() -> None:
|
||||
"""
|
||||
GET /models/status returns rules-only detection mode.
|
||||
"""
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get("/models/status")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["detection_mode"] == "rules-only"
|
||||
assert data["active_models"] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retrain_returns_202() -> None:
|
||||
"""
|
||||
POST /models/retrain returns 202 Accepted with a job ID.
|
||||
"""
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.post("/models/retrain")
|
||||
|
||||
assert response.status_code == 202
|
||||
data = response.json()
|
||||
assert data["status"] == "accepted"
|
||||
assert len(data["job_id"]) == 32
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@
|
|||
test_detection.py
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, UTC
|
||||
|
||||
from app.core.detection.rules import RuleEngine, RuleResult
|
||||
from app.core.detection.rules import RuleEngine
|
||||
from app.core.ingestion.parsers import ParsedLogEntry
|
||||
|
||||
|
||||
|
|
@ -21,7 +21,7 @@ def _make_entry(
|
|||
"""
|
||||
return ParsedLogEntry(
|
||||
ip="93.184.216.34",
|
||||
timestamp=datetime(2026, 2, 11, 14, 30, 0, tzinfo=timezone.utc),
|
||||
timestamp=datetime(2026, 2, 11, 14, 30, 0, tzinfo=UTC),
|
||||
method=method,
|
||||
path=path,
|
||||
query_string=query_string,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
test_features.py
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, UTC
|
||||
|
||||
from app.core.features.extractor import extract_request_features
|
||||
from app.core.ingestion.parsers import ParsedLogEntry
|
||||
|
|
@ -50,7 +50,7 @@ def _make_entry(
|
|||
Build a ParsedLogEntry with sensible defaults for testing.
|
||||
"""
|
||||
if timestamp is None:
|
||||
timestamp = datetime(2026, 2, 11, 14, 30, 0, tzinfo=timezone.utc)
|
||||
timestamp = datetime(2026, 2, 11, 14, 30, 0, tzinfo=UTC)
|
||||
|
||||
return ParsedLogEntry(
|
||||
ip=ip,
|
||||
|
|
@ -80,21 +80,15 @@ def test_path_depth() -> None:
|
|||
"""
|
||||
assert extract_request_features(_make_entry(path="/"))["path_depth"] == 0
|
||||
assert extract_request_features(_make_entry(path="/api"))["path_depth"] == 1
|
||||
assert extract_request_features(
|
||||
_make_entry(path="/api/v1/users")
|
||||
)["path_depth"] == 3
|
||||
assert extract_request_features(_make_entry(path="/api/v1/users"))["path_depth"] == 3
|
||||
|
||||
|
||||
def test_path_entropy_high_vs_low() -> None:
|
||||
"""
|
||||
Random-character paths have higher entropy than simple paths.
|
||||
"""
|
||||
low = extract_request_features(
|
||||
_make_entry(path="/index.html")
|
||||
)["path_entropy"]
|
||||
high = extract_request_features(
|
||||
_make_entry(path="/x8Kp2mQz7wR4vL1n")
|
||||
)["path_entropy"]
|
||||
low = extract_request_features(_make_entry(path="/index.html"))["path_entropy"]
|
||||
high = extract_request_features(_make_entry(path="/x8Kp2mQz7wR4vL1n"))["path_entropy"]
|
||||
assert high > low
|
||||
|
||||
|
||||
|
|
@ -102,9 +96,7 @@ def test_query_string_features() -> None:
|
|||
"""
|
||||
Query param count and length are extracted correctly.
|
||||
"""
|
||||
features = extract_request_features(
|
||||
_make_entry(query_string="page=1&sort=name&limit=50")
|
||||
)
|
||||
features = extract_request_features(_make_entry(query_string="page=1&sort=name&limit=50"))
|
||||
assert features["query_param_count"] == 3
|
||||
assert features["query_string_length"] == len("page=1&sort=name&limit=50")
|
||||
|
||||
|
|
@ -122,9 +114,7 @@ def test_url_encoding_detection() -> None:
|
|||
)
|
||||
assert encoded["has_encoded_chars"] is True
|
||||
|
||||
clean = extract_request_features(
|
||||
_make_entry(path="/index.html", query_string="")
|
||||
)
|
||||
clean = extract_request_features(_make_entry(path="/index.html", query_string=""))
|
||||
assert clean["has_encoded_chars"] is False
|
||||
|
||||
|
||||
|
|
@ -132,14 +122,10 @@ def test_double_encoding_detection() -> None:
|
|||
"""
|
||||
Double-encoded sequences like %2527 are flagged.
|
||||
"""
|
||||
double = extract_request_features(
|
||||
_make_entry(path="/path%2527trick")
|
||||
)
|
||||
double = extract_request_features(_make_entry(path="/path%2527trick"))
|
||||
assert double["has_double_encoding"] is True
|
||||
|
||||
single = extract_request_features(
|
||||
_make_entry(path="/path%27normal")
|
||||
)
|
||||
single = extract_request_features(_make_entry(path="/path%27normal"))
|
||||
assert single["has_double_encoding"] is False
|
||||
|
||||
|
||||
|
|
@ -147,33 +133,23 @@ def test_status_class() -> None:
|
|||
"""
|
||||
Status class groups status codes into Nxx buckets.
|
||||
"""
|
||||
assert extract_request_features(
|
||||
_make_entry(status_code=200)
|
||||
)["status_class"] == "2xx"
|
||||
assert extract_request_features(
|
||||
_make_entry(status_code=404)
|
||||
)["status_class"] == "4xx"
|
||||
assert extract_request_features(
|
||||
_make_entry(status_code=503)
|
||||
)["status_class"] == "5xx"
|
||||
assert extract_request_features(_make_entry(status_code=200))["status_class"] == "2xx"
|
||||
assert extract_request_features(_make_entry(status_code=404))["status_class"] == "4xx"
|
||||
assert extract_request_features(_make_entry(status_code=503))["status_class"] == "5xx"
|
||||
|
||||
|
||||
def test_temporal_features() -> None:
|
||||
"""
|
||||
Hour, day of week, and weekend flag derived from timestamp.
|
||||
"""
|
||||
wednesday_2pm = datetime(2026, 2, 11, 14, 0, 0, tzinfo=timezone.utc)
|
||||
features = extract_request_features(
|
||||
_make_entry(timestamp=wednesday_2pm)
|
||||
)
|
||||
wednesday_2pm = datetime(2026, 2, 11, 14, 0, 0, tzinfo=UTC)
|
||||
features = extract_request_features(_make_entry(timestamp=wednesday_2pm))
|
||||
assert features["hour_of_day"] == 14
|
||||
assert features["day_of_week"] == 2
|
||||
assert features["is_weekend"] is False
|
||||
|
||||
saturday_3am = datetime(2026, 2, 14, 3, 0, 0, tzinfo=timezone.utc)
|
||||
weekend = extract_request_features(
|
||||
_make_entry(timestamp=saturday_3am)
|
||||
)
|
||||
saturday_3am = datetime(2026, 2, 14, 3, 0, 0, tzinfo=UTC)
|
||||
weekend = extract_request_features(_make_entry(timestamp=saturday_3am))
|
||||
assert weekend["hour_of_day"] == 3
|
||||
assert weekend["day_of_week"] == 5
|
||||
assert weekend["is_weekend"] is True
|
||||
|
|
@ -200,14 +176,10 @@ def test_ua_scanner_detection() -> None:
|
|||
"""
|
||||
Known vulnerability scanner user agents are flagged.
|
||||
"""
|
||||
nikto = extract_request_features(
|
||||
_make_entry(user_agent="Mozilla/5.00 (Nikto/2.1.6)")
|
||||
)
|
||||
nikto = extract_request_features(_make_entry(user_agent="Mozilla/5.00 (Nikto/2.1.6)"))
|
||||
assert nikto["is_known_scanner"] is True
|
||||
|
||||
sqlmap = extract_request_features(
|
||||
_make_entry(user_agent="sqlmap/1.8")
|
||||
)
|
||||
sqlmap = extract_request_features(_make_entry(user_agent="sqlmap/1.8"))
|
||||
assert sqlmap["is_known_scanner"] is True
|
||||
|
||||
|
||||
|
|
@ -215,9 +187,7 @@ def test_attack_pattern_detection() -> None:
|
|||
"""
|
||||
SQLi, XSS, and path traversal patterns in paths are detected.
|
||||
"""
|
||||
sqli = extract_request_features(
|
||||
_make_entry(path="/users", query_string="id=1' OR 1=1--")
|
||||
)
|
||||
sqli = extract_request_features(_make_entry(path="/users", query_string="id=1' OR 1=1--"))
|
||||
assert sqli["has_attack_pattern"] is True
|
||||
|
||||
xss = extract_request_features(
|
||||
|
|
@ -225,14 +195,10 @@ def test_attack_pattern_detection() -> None:
|
|||
)
|
||||
assert xss["has_attack_pattern"] is True
|
||||
|
||||
traversal = extract_request_features(
|
||||
_make_entry(path="/static/../../etc/passwd")
|
||||
)
|
||||
traversal = extract_request_features(_make_entry(path="/static/../../etc/passwd"))
|
||||
assert traversal["has_attack_pattern"] is True
|
||||
|
||||
clean = extract_request_features(
|
||||
_make_entry(path="/api/v1/health")
|
||||
)
|
||||
clean = extract_request_features(_make_entry(path="/api/v1/health"))
|
||||
assert clean["has_attack_pattern"] is False
|
||||
|
||||
|
||||
|
|
@ -240,13 +206,11 @@ def test_special_char_ratio() -> None:
|
|||
"""
|
||||
Paths with many non-alphanumeric characters have higher ratios.
|
||||
"""
|
||||
clean = extract_request_features(
|
||||
_make_entry(path="/api/users")
|
||||
)["special_char_ratio"]
|
||||
clean = extract_request_features(_make_entry(path="/api/users"))["special_char_ratio"]
|
||||
|
||||
noisy = extract_request_features(
|
||||
_make_entry(path="/<script>alert('xss')</script>")
|
||||
)["special_char_ratio"]
|
||||
noisy = extract_request_features(_make_entry(path="/<script>alert('xss')</script>"))[
|
||||
"special_char_ratio"
|
||||
]
|
||||
|
||||
assert noisy > clean
|
||||
|
||||
|
|
@ -255,51 +219,39 @@ def test_private_ip_detection() -> None:
|
|||
"""
|
||||
RFC 1918 and loopback addresses are classified as private.
|
||||
"""
|
||||
assert extract_request_features(
|
||||
_make_entry(ip="192.168.1.1")
|
||||
)["is_private_ip"] is True
|
||||
assert extract_request_features(_make_entry(ip="192.168.1.1"))["is_private_ip"] is True
|
||||
|
||||
assert extract_request_features(
|
||||
_make_entry(ip="127.0.0.1")
|
||||
)["is_private_ip"] is True
|
||||
assert extract_request_features(_make_entry(ip="127.0.0.1"))["is_private_ip"] is True
|
||||
|
||||
assert extract_request_features(
|
||||
_make_entry(ip="8.8.8.8")
|
||||
)["is_private_ip"] is False
|
||||
assert extract_request_features(_make_entry(ip="8.8.8.8"))["is_private_ip"] is False
|
||||
|
||||
|
||||
def test_file_extension() -> None:
|
||||
"""
|
||||
File extension is extracted from the path.
|
||||
"""
|
||||
assert extract_request_features(
|
||||
_make_entry(path="/style.css")
|
||||
)["file_extension"] == ".css"
|
||||
assert extract_request_features(_make_entry(path="/style.css"))["file_extension"] == ".css"
|
||||
|
||||
assert extract_request_features(
|
||||
_make_entry(path="/api/users")
|
||||
)["file_extension"] == ""
|
||||
assert extract_request_features(_make_entry(path="/api/users"))["file_extension"] == ""
|
||||
|
||||
|
||||
def test_country_code_passthrough() -> None:
|
||||
"""
|
||||
Country code is passed through from the caller.
|
||||
"""
|
||||
features = extract_request_features(
|
||||
_make_entry(), country_code="US"
|
||||
)
|
||||
features = extract_request_features(_make_entry(), country_code="US")
|
||||
assert features["country_code"] == "US"
|
||||
|
||||
features_empty = extract_request_features(_make_entry())
|
||||
assert features_empty["country_code"] == ""
|
||||
|
||||
|
||||
import time
|
||||
import time # noqa: E402
|
||||
|
||||
import fakeredis.aioredis
|
||||
import pytest
|
||||
import fakeredis.aioredis # noqa: E402
|
||||
import pytest # noqa: E402
|
||||
|
||||
from app.core.features.aggregator import WindowAggregator
|
||||
from app.core.features.aggregator import WindowAggregator # noqa: E402
|
||||
|
||||
AGGREGATOR_KEYS = {
|
||||
"req_count_1m",
|
||||
|
|
@ -476,8 +428,8 @@ async def test_aggregator_window_boundary(aggregator) -> None:
|
|||
assert result["req_count_5m"] == 2
|
||||
|
||||
|
||||
from app.core.features.encoder import encode_for_inference
|
||||
from app.core.features.mappings import FEATURE_ORDER, METHOD_MAP, STATUS_CLASS_MAP
|
||||
from app.core.features.encoder import encode_for_inference # noqa: E402
|
||||
from app.core.features.mappings import FEATURE_ORDER, METHOD_MAP, STATUS_CLASS_MAP # noqa: E402
|
||||
|
||||
|
||||
def _full_features() -> dict[str, int | float | bool | str]:
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
test_geoip.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -109,9 +108,7 @@ async def test_lookup_missing_city_name(mock_reader) -> None:
|
|||
"""
|
||||
Responses with None city name are handled gracefully.
|
||||
"""
|
||||
mock_reader.city.return_value = _mock_city_response(
|
||||
city_name=None, lat=0.0, lon=0.0
|
||||
)
|
||||
mock_reader.city.return_value = _mock_city_response(city_name=None, lat=0.0, lon=0.0)
|
||||
service = GeoIPService.__new__(GeoIPService)
|
||||
service._reader = mock_reader
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,199 @@
|
|||
"""
|
||||
©AngelaMos | 2026
|
||||
test_integration.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fakeredis.aioredis import FakeRedis
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
from sqlmodel import SQLModel
|
||||
|
||||
from app.core.alerts.dispatcher import AlertDispatcher
|
||||
from app.core.detection.rules import RuleEngine
|
||||
from app.core.ingestion.pipeline import Pipeline
|
||||
from app.core.ingestion.tailer import LogTailer
|
||||
from app.models.threat_event import ThreatEvent
|
||||
|
||||
NORMAL_LINE = (
|
||||
"192.168.1.100 - - [11/Feb/2026:10:00:00 +0000] "
|
||||
'"GET /index.html HTTP/1.1" 200 4523 "-" '
|
||||
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"'
|
||||
)
|
||||
|
||||
SQLI_LINE = (
|
||||
"198.51.100.10 - - [11/Feb/2026:10:00:01 +0000] "
|
||||
'"GET /search?q=1%27+OR+1=1-- HTTP/1.1" 200 5678 "-" '
|
||||
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"'
|
||||
)
|
||||
|
||||
XSS_LINE = (
|
||||
"198.51.100.11 - - [11/Feb/2026:10:00:02 +0000] "
|
||||
'"GET /comment?text=<script>alert(1)</script> HTTP/1.1" 200 3210 "-" '
|
||||
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"'
|
||||
)
|
||||
|
||||
PATH_TRAVERSAL_LINE = (
|
||||
"198.51.100.12 - - [11/Feb/2026:10:00:03 +0000] "
|
||||
'"GET /../../etc/passwd HTTP/1.1" 400 230 "-" '
|
||||
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"'
|
||||
)
|
||||
|
||||
|
||||
def _write_lines(log_path: str, *lines: str) -> None:
|
||||
"""
|
||||
Append lines to the log file and force an OS-level flush
|
||||
so inotify fires immediately.
|
||||
"""
|
||||
with open(log_path, "a") as f:
|
||||
for line in lines:
|
||||
f.write(line + "\n")
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def integration_env():
|
||||
"""
|
||||
Full-stack integration environment with in-memory DB,
|
||||
fake Redis, pipeline, and temp log directory.
|
||||
"""
|
||||
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
log_path = os.path.join(tmp_dir, "access.log")
|
||||
Path(log_path).touch()
|
||||
|
||||
fake_redis = FakeRedis(decode_responses=True)
|
||||
engine = create_async_engine(
|
||||
"sqlite+aiosqlite://",
|
||||
connect_args={"check_same_thread": False},
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(SQLModel.metadata.create_all)
|
||||
|
||||
session_factory = async_sessionmaker(
|
||||
engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False,
|
||||
)
|
||||
|
||||
dispatcher = AlertDispatcher(fake_redis, session_factory)
|
||||
rule_engine = RuleEngine()
|
||||
|
||||
pipeline = Pipeline(
|
||||
redis_client=fake_redis,
|
||||
rule_engine=rule_engine,
|
||||
on_result=dispatcher.dispatch,
|
||||
)
|
||||
await pipeline.start()
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
tailer = LogTailer(log_path, pipeline.raw_queue, loop)
|
||||
tailer.start()
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
yield {
|
||||
"log_path": log_path,
|
||||
"pipeline": pipeline,
|
||||
"tailer": tailer,
|
||||
"session_factory": session_factory,
|
||||
"engine": engine,
|
||||
}
|
||||
|
||||
tailer.stop()
|
||||
await pipeline.stop()
|
||||
await engine.dispose()
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
|
||||
|
||||
async def _poll_threat_count(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
expected: int,
|
||||
timeout: float = 8.0,
|
||||
) -> int:
|
||||
"""
|
||||
Poll the database until the expected threat count is reached or timeout.
|
||||
"""
|
||||
count = 0
|
||||
for _ in range(int(timeout / 0.1)):
|
||||
await asyncio.sleep(0.1)
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(select(func.count()).select_from(ThreatEvent))
|
||||
count = result.scalar_one()
|
||||
if count >= expected:
|
||||
return count
|
||||
return count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tailer_to_db_end_to_end(integration_env) -> None:
|
||||
"""
|
||||
Write log lines to a file - tailer picks them up - pipeline processes -
|
||||
dispatcher stores MEDIUM+ threats in the database.
|
||||
"""
|
||||
env = integration_env
|
||||
_write_lines(
|
||||
env["log_path"],
|
||||
NORMAL_LINE,
|
||||
SQLI_LINE,
|
||||
XSS_LINE,
|
||||
PATH_TRAVERSAL_LINE,
|
||||
)
|
||||
|
||||
count = await _poll_threat_count(env["session_factory"], expected=3)
|
||||
assert count >= 3, f"Expected >= 3 stored threats, got {count}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_only_medium_plus_stored(integration_env) -> None:
|
||||
"""
|
||||
Normal (LOW severity) requests are NOT stored in the database.
|
||||
Only MEDIUM and HIGH severity threats are persisted.
|
||||
"""
|
||||
env = integration_env
|
||||
lines = [
|
||||
f"192.168.1.{i + 1} - - [11/Feb/2026:10:00:0{i} +0000] "
|
||||
f'"GET /page/{i} HTTP/1.1" 200 1234 "-" '
|
||||
f'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"'
|
||||
for i in range(5)
|
||||
]
|
||||
_write_lines(env["log_path"], *lines)
|
||||
|
||||
await asyncio.sleep(2.0)
|
||||
|
||||
async with env["session_factory"]() as session:
|
||||
result = await session.execute(select(func.count()).select_from(ThreatEvent))
|
||||
count = result.scalar_one()
|
||||
|
||||
assert count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stored_threats_have_correct_fields(integration_env) -> None:
|
||||
"""
|
||||
Stored threat events have populated severity, score, and matched rules.
|
||||
"""
|
||||
env = integration_env
|
||||
_write_lines(env["log_path"], SQLI_LINE)
|
||||
|
||||
count = await _poll_threat_count(env["session_factory"], expected=1)
|
||||
assert count >= 1, f"Expected >= 1 stored threat, got {count}"
|
||||
|
||||
async with env["session_factory"]() as session:
|
||||
rows = (await session.execute(select(ThreatEvent))).scalars().all()
|
||||
|
||||
event = rows[0]
|
||||
assert event.severity in ("HIGH", "MEDIUM")
|
||||
assert event.threat_score >= 0.5
|
||||
assert len(event.matched_rules) > 0
|
||||
assert len(event.feature_vector) == 35
|
||||
assert event.source_ip == "198.51.100.10"
|
||||
|
|
@ -3,7 +3,7 @@
|
|||
test_parsers.py
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, UTC
|
||||
|
||||
from app.core.ingestion.parsers import ParsedLogEntry, parse_combined
|
||||
|
||||
|
|
@ -13,7 +13,7 @@ def test_parse_standard_combined_line() -> None:
|
|||
Parse a standard nginx combined log line into all fields.
|
||||
"""
|
||||
line = (
|
||||
'93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] '
|
||||
"93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] "
|
||||
'"GET /api/users?page=1 HTTP/1.1" 200 1234 '
|
||||
'"https://example.com" '
|
||||
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"'
|
||||
|
|
@ -23,7 +23,7 @@ def test_parse_standard_combined_line() -> None:
|
|||
assert result is not None
|
||||
assert isinstance(result, ParsedLogEntry)
|
||||
assert result.ip == "93.184.216.34"
|
||||
assert result.timestamp == datetime(2026, 2, 11, 14, 30, 0, tzinfo=timezone.utc)
|
||||
assert result.timestamp == datetime(2026, 2, 11, 14, 30, 0, tzinfo=UTC)
|
||||
assert result.method == "GET"
|
||||
assert result.path == "/api/users"
|
||||
assert result.query_string == "page=1"
|
||||
|
|
@ -38,10 +38,7 @@ def test_parse_ipv6_address() -> None:
|
|||
"""
|
||||
Parse a line with an IPv6 source address.
|
||||
"""
|
||||
line = (
|
||||
'::1 - - [11/Feb/2026:14:30:00 +0000] '
|
||||
'"GET / HTTP/1.1" 200 612 "-" "curl/8.0"'
|
||||
)
|
||||
line = '::1 - - [11/Feb/2026:14:30:00 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/8.0"'
|
||||
result = parse_combined(line)
|
||||
|
||||
assert result is not None
|
||||
|
|
@ -56,7 +53,7 @@ def test_parse_missing_referer() -> None:
|
|||
A dash referer is normalized to an empty string.
|
||||
"""
|
||||
line = (
|
||||
'10.0.0.1 - - [11/Feb/2026:08:15:42 +0000] '
|
||||
"10.0.0.1 - - [11/Feb/2026:08:15:42 +0000] "
|
||||
'"POST /login HTTP/1.1" 302 0 "-" "Mozilla/5.0"'
|
||||
)
|
||||
result = parse_combined(line)
|
||||
|
|
@ -72,7 +69,7 @@ def test_parse_complex_query_string() -> None:
|
|||
Query strings with multiple parameters and special characters.
|
||||
"""
|
||||
line = (
|
||||
'93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] '
|
||||
"93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] "
|
||||
'"GET /search?q=hello+world&lang=en&page=2&sort=relevance HTTP/1.1" '
|
||||
'200 5678 "https://example.com/search" "Mozilla/5.0"'
|
||||
)
|
||||
|
|
@ -101,10 +98,7 @@ def test_parse_dash_response_size() -> None:
|
|||
"""
|
||||
A dash response size (e.g. HEAD 304) is normalized to zero.
|
||||
"""
|
||||
line = (
|
||||
'1.2.3.4 - - [11/Feb/2026:10:00:00 +0000] '
|
||||
'"HEAD / HTTP/1.1" 304 - "-" "Mozilla/5.0"'
|
||||
)
|
||||
line = '1.2.3.4 - - [11/Feb/2026:10:00:00 +0000] "HEAD / HTTP/1.1" 304 - "-" "Mozilla/5.0"'
|
||||
result = parse_combined(line)
|
||||
|
||||
assert result is not None
|
||||
|
|
@ -117,7 +111,7 @@ def test_parse_full_ipv6_address() -> None:
|
|||
Parse a line with a full-length IPv6 address.
|
||||
"""
|
||||
line = (
|
||||
'2001:0db8:85a3:0000:0000:8a2e:0370:7334 - - '
|
||||
"2001:0db8:85a3:0000:0000:8a2e:0370:7334 - - "
|
||||
'[11/Feb/2026:14:30:00 +0000] "GET /api/v1/health HTTP/2.0" '
|
||||
'200 256 "-" "python-httpx/0.28"'
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,8 +3,6 @@
|
|||
test_pipeline.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
import fakeredis.aioredis
|
||||
import pytest
|
||||
|
||||
|
|
@ -12,14 +10,14 @@ from app.core.detection.rules import RuleEngine
|
|||
from app.core.ingestion.pipeline import Pipeline, ScoredRequest
|
||||
|
||||
VALID_LINE = (
|
||||
'93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] '
|
||||
"93.184.216.34 - - [11/Feb/2026:14:30:00 +0000] "
|
||||
'"GET /api/v1/users HTTP/1.1" 200 1234 '
|
||||
'"https://example.com" '
|
||||
'"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"'
|
||||
)
|
||||
|
||||
SQLI_LINE = (
|
||||
'93.184.216.34 - - [11/Feb/2026:14:30:01 +0000] '
|
||||
"93.184.216.34 - - [11/Feb/2026:14:30:01 +0000] "
|
||||
'"GET /users?id=1%27OR%201=1-- HTTP/1.1" 200 512 '
|
||||
'"-" "Mozilla/5.0"'
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
192.168.1.10 - - [11/Feb/2026:08:00:01 +0000] "GET /index.html HTTP/1.1" 200 4523 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
192.168.1.10 - - [11/Feb/2026:08:00:02 +0000] "GET /static/css/main.css HTTP/1.1" 200 8921 "http://example.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
192.168.1.10 - - [11/Feb/2026:08:00:02 +0000] "GET /static/js/app.js HTTP/1.1" 200 45230 "http://example.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
192.168.1.10 - - [11/Feb/2026:08:00:03 +0000] "GET /static/img/logo.png HTTP/1.1" 304 0 "http://example.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
10.0.0.50 - - [11/Feb/2026:08:01:00 +0000] "GET /api/v1/users HTTP/1.1" 200 2340 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"
|
||||
10.0.0.50 - - [11/Feb/2026:08:01:01 +0000] "POST /api/v1/auth/login HTTP/1.1" 200 312 "http://example.com/login" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"
|
||||
10.0.0.50 - - [11/Feb/2026:08:01:02 +0000] "GET /api/v1/dashboard HTTP/1.1" 200 8920 "http://example.com/dashboard" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"
|
||||
172.16.0.5 - - [11/Feb/2026:08:02:00 +0000] "GET / HTTP/1.1" 301 162 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0"
|
||||
172.16.0.5 - - [11/Feb/2026:08:02:01 +0000] "GET /index.html HTTP/1.1" 200 4523 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0"
|
||||
172.16.0.5 - - [11/Feb/2026:08:02:05 +0000] "GET /about HTTP/1.1" 200 3210 "http://example.com/" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0"
|
||||
172.16.0.5 - - [11/Feb/2026:08:02:06 +0000] "GET /contact HTTP/1.1" 200 2100 "http://example.com/about" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0"
|
||||
192.168.1.20 - - [11/Feb/2026:08:03:00 +0000] "GET /products HTTP/1.1" 200 12450 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"
|
||||
192.168.1.20 - - [11/Feb/2026:08:03:01 +0000] "GET /products/42 HTTP/1.1" 200 3210 "http://example.com/products" "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"
|
||||
192.168.1.20 - - [11/Feb/2026:08:03:02 +0000] "POST /api/v1/cart HTTP/1.1" 201 128 "http://example.com/products/42" "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"
|
||||
192.168.1.20 - - [11/Feb/2026:08:03:05 +0000] "GET /api/v1/cart HTTP/1.1" 200 456 "http://example.com/cart" "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"
|
||||
10.0.0.100 - - [11/Feb/2026:08:04:00 +0000] "GET /robots.txt HTTP/1.1" 200 230 "-" "Googlebot/2.1 (+http://www.google.com/bot.html)"
|
||||
10.0.0.100 - - [11/Feb/2026:08:04:01 +0000] "GET /sitemap.xml HTTP/1.1" 200 9800 "-" "Googlebot/2.1 (+http://www.google.com/bot.html)"
|
||||
10.0.0.101 - - [11/Feb/2026:08:05:00 +0000] "GET / HTTP/1.1" 200 4523 "-" "Bingbot/2.0 (+http://www.bing.com/bingbot.htm)"
|
||||
10.0.0.102 - - [11/Feb/2026:08:06:00 +0000] "HEAD / HTTP/1.1" 200 0 "-" "Mozilla/5.0 (compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)"
|
||||
172.16.0.10 - - [11/Feb/2026:08:07:00 +0000] "GET /api/v1/health HTTP/1.1" 200 89 "-" "curl/8.7.1"
|
||||
172.16.0.10 - - [11/Feb/2026:08:07:01 +0000] "GET /api/v1/ready HTTP/1.1" 200 145 "-" "curl/8.7.1"
|
||||
192.168.1.30 - - [11/Feb/2026:08:08:00 +0000] "GET /blog HTTP/1.1" 200 15600 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
192.168.1.30 - - [11/Feb/2026:08:08:02 +0000] "GET /blog/post/hello-world HTTP/1.1" 200 8900 "http://example.com/blog" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
192.168.1.30 - - [11/Feb/2026:08:08:05 +0000] "POST /blog/post/hello-world/comment HTTP/1.1" 201 256 "http://example.com/blog/post/hello-world" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
10.0.0.55 - - [11/Feb/2026:08:09:00 +0000] "GET /docs HTTP/1.1" 200 23400 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
10.0.0.55 - - [11/Feb/2026:08:09:01 +0000] "GET /docs/api HTTP/1.1" 200 12300 "http://example.com/docs" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
10.0.0.55 - - [11/Feb/2026:08:09:02 +0000] "GET /docs/quickstart HTTP/1.1" 200 9800 "http://example.com/docs" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
192.168.1.40 - - [11/Feb/2026:08:10:00 +0000] "GET /favicon.ico HTTP/1.1" 200 1150 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
192.168.1.40 - - [11/Feb/2026:08:10:01 +0000] "GET /static/fonts/roboto.woff2 HTTP/1.1" 200 45000 "http://example.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
172.16.0.20 - - [11/Feb/2026:08:11:00 +0000] "GET /nonexistent-page HTTP/1.1" 404 1234 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0"
|
||||
172.16.0.20 - - [11/Feb/2026:08:11:01 +0000] "GET /old-page HTTP/1.1" 301 162 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0"
|
||||
10.0.0.60 - - [11/Feb/2026:08:12:00 +0000] "OPTIONS /api/v1/users HTTP/1.1" 204 0 "http://frontend.example.com" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
10.0.0.60 - - [11/Feb/2026:08:12:01 +0000] "PUT /api/v1/users/5 HTTP/1.1" 200 512 "http://frontend.example.com/settings" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
10.0.0.60 - - [11/Feb/2026:08:12:02 +0000] "DELETE /api/v1/cart/item/7 HTTP/1.1" 204 0 "http://frontend.example.com/cart" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
192.168.1.50 - - [11/Feb/2026:08:13:00 +0000] "GET /search?q=wireless+keyboard HTTP/1.1" 200 6700 "http://example.com/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"
|
||||
192.168.1.50 - - [11/Feb/2026:08:13:01 +0000] "GET /search?q=usb+hub&page=2 HTTP/1.1" 200 5400 "http://example.com/search?q=wireless+keyboard" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"
|
||||
10.0.0.70 - - [11/Feb/2026:08:14:00 +0000] "GET /api/v1/orders HTTP/1.1" 401 89 "-" "python-requests/2.32.0"
|
||||
10.0.0.70 - - [11/Feb/2026:08:14:01 +0000] "POST /api/v1/auth/token HTTP/1.1" 200 256 "-" "python-requests/2.32.0"
|
||||
10.0.0.70 - - [11/Feb/2026:08:14:02 +0000] "GET /api/v1/orders HTTP/1.1" 200 3400 "-" "python-requests/2.32.0"
|
||||
192.168.1.60 - - [11/Feb/2026:08:15:00 +0000] "GET /static/img/hero.jpg HTTP/1.1" 200 250000 "http://example.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
203.0.113.50 - - [11/Feb/2026:08:20:00 +0000] "GET / HTTP/1.1" 200 4523 "-" "Nikto/2.1.6"
|
||||
203.0.113.50 - - [11/Feb/2026:08:20:01 +0000] "GET /admin HTTP/1.1" 404 1234 "-" "Nikto/2.1.6"
|
||||
203.0.113.50 - - [11/Feb/2026:08:20:02 +0000] "GET /phpmyadmin HTTP/1.1" 404 1234 "-" "Nikto/2.1.6"
|
||||
203.0.113.51 - - [11/Feb/2026:08:21:00 +0000] "GET /.env HTTP/1.1" 404 1234 "-" "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)"
|
||||
203.0.113.51 - - [11/Feb/2026:08:21:01 +0000] "GET /wp-login.php HTTP/1.1" 404 1234 "-" "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)"
|
||||
198.51.100.10 - - [11/Feb/2026:08:30:00 +0000] "GET /search?q=1%27+OR+1=1-- HTTP/1.1" 200 5678 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
198.51.100.11 - - [11/Feb/2026:08:31:00 +0000] "GET /comment?text=<script>alert(document.cookie)</script> HTTP/1.1" 200 3210 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
198.51.100.12 - - [11/Feb/2026:08:32:00 +0000] "GET /../../etc/passwd HTTP/1.1" 400 230 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
198.51.100.13 - - [11/Feb/2026:08:33:00 +0000] "GET /api/ping?host=;cat /etc/passwd HTTP/1.1" 200 1200 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
198.51.100.14 - - [11/Feb/2026:08:34:00 +0000] "GET /page?file=php://filter/convert.base64-encode/resource=index HTTP/1.1" 200 8900 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# ©AngelaMos | 2026
|
||||
# Development Docker Compose
|
||||
# dev.compose.yml
|
||||
|
||||
services:
|
||||
postgres:
|
||||
|
|
@ -8,90 +8,74 @@ services:
|
|||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB:-angelusvigil}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-vigil}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD required}
|
||||
POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C"
|
||||
volumes:
|
||||
- postgres_dev_data:/var/lib/postgresql/data
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-devpassword}
|
||||
ports:
|
||||
- "${POSTGRES_HOST_PORT:-16969}:5432"
|
||||
volumes:
|
||||
- postgres_dev:/var/lib/postgresql/data
|
||||
networks:
|
||||
- vigil_network_dev
|
||||
- vigil_dev
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-vigil} -d ${POSTGRES_DB:-angelusvigil}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
start_period: 10s
|
||||
|
||||
redis:
|
||||
image: redis:7.4-alpine
|
||||
container_name: vigil-redis-dev
|
||||
command: redis-server /usr/local/etc/redis/redis.conf
|
||||
volumes:
|
||||
- redis_dev_data:/data
|
||||
- ./infra/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro
|
||||
command: redis-server --appendonly yes
|
||||
ports:
|
||||
- "${REDIS_HOST_PORT:-26969}:6379"
|
||||
volumes:
|
||||
- redis_dev:/data
|
||||
networks:
|
||||
- vigil_network_dev
|
||||
- vigil_dev
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
start_period: 3s
|
||||
|
||||
backend:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/docker/fastapi.dev
|
||||
dockerfile: infra/docker/fastapi.prod
|
||||
container_name: vigil-backend-dev
|
||||
environment:
|
||||
ENV: ${ENV:-development}
|
||||
DEBUG: ${DEBUG:-true}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-DEBUG}
|
||||
API_KEY: ${API_KEY:-dev-api-key}
|
||||
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-vigil}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-angelusvigil}
|
||||
REDIS_URL: ${REDIS_URL:-redis://redis:6379}
|
||||
ENV: development
|
||||
DEBUG: "true"
|
||||
LOG_LEVEL: INFO
|
||||
API_KEY: ${API_KEY:-dev-test-key}
|
||||
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-vigil}:${POSTGRES_PASSWORD:-devpassword}@postgres:5432/${POSTGRES_DB:-angelusvigil}
|
||||
REDIS_URL: redis://redis:6379
|
||||
NGINX_LOG_PATH: /var/log/nginx/access.log
|
||||
GEOIP_DB_PATH: /usr/share/GeoIP/GeoLite2-City.mmdb
|
||||
ports:
|
||||
- "${BACKEND_HOST_PORT:-36969}:8000"
|
||||
volumes:
|
||||
- ./backend:/app
|
||||
- /app/.venv
|
||||
- /app/__pycache__
|
||||
- ./data/sample-logs:/var/log/nginx:ro
|
||||
- geoip_dev_data:/usr/share/GeoIP:ro
|
||||
- nginx_logs_dev:/var/log/nginx
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- vigil_network_dev
|
||||
ports:
|
||||
- "${BACKEND_HOST_PORT:-36969}:8000"
|
||||
restart: unless-stopped
|
||||
|
||||
geoip-updater:
|
||||
image: maxmindinc/geoipupdate:latest
|
||||
container_name: vigil-geoip-dev
|
||||
environment:
|
||||
GEOIPUPDATE_ACCOUNT_ID: ${GEOIP_ACCOUNT_ID:-0}
|
||||
GEOIPUPDATE_LICENSE_KEY: ${GEOIP_LICENSE_KEY:-placeholder}
|
||||
GEOIPUPDATE_EDITION_IDS: "GeoLite2-City"
|
||||
GEOIPUPDATE_FREQUENCY: "168"
|
||||
volumes:
|
||||
- geoip_dev_data:/usr/share/GeoIP
|
||||
networks:
|
||||
- vigil_network_dev
|
||||
- vigil_dev
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 15s
|
||||
|
||||
networks:
|
||||
vigil_network_dev:
|
||||
vigil_dev:
|
||||
driver: bridge
|
||||
name: vigil_network_dev
|
||||
|
||||
volumes:
|
||||
postgres_dev_data:
|
||||
name: vigil_postgres_dev_data
|
||||
redis_dev_data:
|
||||
name: vigil_redis_dev_data
|
||||
geoip_dev_data:
|
||||
name: vigil_geoip_dev_data
|
||||
postgres_dev:
|
||||
redis_dev:
|
||||
nginx_logs_dev:
|
||||
|
|
|
|||
|
|
@ -18,7 +18,8 @@ RUN apt-get update && \
|
|||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY backend/pyproject.toml ./
|
||||
RUN uv pip install --target /app/deps .
|
||||
RUN uv pip compile pyproject.toml -o requirements.txt && \
|
||||
uv pip install --target /app/deps -r requirements.txt
|
||||
|
||||
FROM python:3.14-slim
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue