diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index eff214e1..3831982f 100755 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,7 +19,7 @@ jobs: defaults: run: - working-directory: PROJECTS/api-security-scanner/backend + working-directory: PROJECTS/backend steps: - name: Checkout code @@ -34,7 +34,7 @@ jobs: uses: actions/cache@v4 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('PROJECTS/api-security-scanner/backend/pyproject.toml') }} + key: ${{ runner.os }}-pip-${{ hashFiles('PROJECTS/backend/pyproject.toml') }} restore-keys: | ${{ runner.os }}-pip- @@ -49,10 +49,10 @@ jobs: echo "Running pylint..." if pylint . > pylint-output.txt 2>&1; then echo "PYLINT_PASSED=true" >> $GITHUB_ENV - echo "✅ No pylint errors found!" + echo "No pylint errors found!" else echo "PYLINT_PASSED=false" >> $GITHUB_ENV - echo "⚠️ Pylint found issues!" + echo "Pylint found issues!" fi cat pylint-output.txt continue-on-error: true @@ -63,10 +63,10 @@ jobs: echo "Running ruff check..." if ruff check . > ruff-output.txt 2>&1; then echo "RUFF_PASSED=true" >> $GITHUB_ENV - echo "✅ No ruff errors found!" + echo "No ruff errors found!" else echo "RUFF_PASSED=false" >> $GITHUB_ENV - echo "⚠️ Ruff found issues" + echo "Ruff found issues" fi cat ruff-output.txt continue-on-error: true @@ -77,10 +77,10 @@ jobs: echo "Running mypy..." if mypy . > mypy-output.txt 2>&1; then echo "MYPY_PASSED=true" >> $GITHUB_ENV - echo "✅ No mypy errors found" + echo "No mypy errors found" else echo "MYPY_PASSED=false" >> $GITHUB_ENV - echo "⚠️ Mypy found issues" + echo "Mypy found issues" fi cat mypy-output.txt continue-on-error: true @@ -90,15 +90,15 @@ jobs: if: github.event_name == 'pull_request' run: | { - echo '## 🔍 Lint & Type Check Results' + echo '##Lint & Type Check Results' echo '' # Pylint Status if [[ "${{ env.PYLINT_PASSED }}" == "true" ]]; then - echo '### ✅ Pylint: **Passed**' + echo '###Pylint: **Passed**' echo 'No pylint issues found.' else - echo '### ⚠️ Pylint: **Issues Found**' + echo '###Pylint: **Issues Found**' echo '
View pylint output' echo '' echo '```' @@ -110,7 +110,7 @@ jobs: # Ruff Status if [[ "${{ env.RUFF_PASSED }}" == "true" ]]; then - echo '### ✅ Ruff: **Passed**' + echo '###Ruff: **Passed**' echo 'No ruff issues found.' else echo '### ⚠️ Ruff: **Issues Found**' @@ -125,10 +125,10 @@ jobs: # Mypy Status if [[ "${{ env.MYPY_PASSED }}" == "true" ]]; then - echo '### ✅ Mypy: **Passed**' + echo '### Mypy: **Passed**' echo 'No mypy issues found.' else - echo '### ⚠️ Mypy: **Issues Found**' + echo '### Mypy: **Issues Found**' echo '
View mypy output' echo '' echo '```' diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..07c4b1c3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +cyber-proj +*.cache diff --git a/PROJECTS/Aenebris/.angela/NEXT_websockets-prep.md b/PROJECTS/Aenebris/.angela/NEXT_websockets-prep.md new file mode 100644 index 00000000..00375aec --- /dev/null +++ b/PROJECTS/Aenebris/.angela/NEXT_websockets-prep.md @@ -0,0 +1,227 @@ +# WebSocket Implementation - Research & Prep (Milestone 1.5) + +**Created:** 2025-11-16 +**Status:** Planning Phase + +--- + +## 🎯 **The REAL Goal** + +Not just "make WebSockets work" - that's easy. +**Make WebSockets BETTER than nginx** and **solve the streaming/WebSocket conflict** that nginx has. + +### **The Problem We're Solving** (From Real Experience) + +> "streaming doesn't even work for me, and I remember hearing a while ago I have no clue if true but the thing to allow websockets to work conflict with what makes streaming work in nginx so I can't have both working" + +**THIS is our target:** Make WebSockets + Streaming work SIMULTANEOUSLY without conflicts. + +Nginx config issues: +- `proxy_buffering off` needed for streaming (Ollama AI) +- `proxy_http_version 1.1` + `Upgrade` + `Connection "upgrade"` needed for WebSockets +- Timeouts configured differently (`proxy_read_timeout 86400` for WebSockets vs `300s` for streaming) +- They conflict in nginx - can't have both perfect + +**Our advantage:** We're building from scratch in Haskell - we can do BOTH right by design! + +--- + +## 🧠 **Key Questions to Research** + +### 1. **Technical Questions** +- How does WAI/Warp handle WebSocket upgrades natively? +- Does Haskell's `websockets` library play nice with Warp? +- How do we detect WebSocket handshake vs regular HTTP? +- Can we proxy WebSocket frames without parsing them? (performance) +- How do we handle bidirectional streaming efficiently? (use STM channels? conduits?) + +### 2. **The Conflict Question** ⚠️ +- WHY do streaming and WebSockets conflict in nginx? +- Is it buffering? Connection reuse? Timeout handling? +- How do we architect to avoid this entirely? + +### 3. **Performance Questions** +- What's the overhead of proxying WebSocket frames? +- Can we do zero-copy WebSocket proxying? +- How do we handle thousands of concurrent WebSocket connections? +- Memory usage per connection? + +### 4. **Protocol Questions** +- WebSocket handshake: do we validate or just proxy it? +- Do we need to parse WebSocket frames or pass through opaque? +- How do we handle WebSocket extensions (compression, etc.)? +- ping/pong frame handling - proxy or handle ourselves? + +--- + +## 📚 **Research Needed** + +### **Existing Research Docs** +We have: +- ✅ `http2-http3.md` - might have streaming info +- ✅ `performance-optimization.md` - zero-copy, memory efficiency +- ✅ `tls-ssl.md` - secure WebSockets (wss://) + +Need to READ for relevant info. + +### **New Research Required** + +#### **1. Modern WebSocket Best Practices (2024/2025)** +Sources to search: +- RFC 6455 (WebSocket protocol) - official spec +- Haskell `websockets` library docs - latest version +- Warp WebSocket examples - production patterns +- Stack Overflow - "nginx websocket streaming conflict" - find the root cause! +- Modern WebSocket proxying techniques - what's changed since 2020? + +#### **2. Haskell-Specific Patterns** +- How does `websockets` library integrate with WAI? +- Conduit vs Pipes vs Streaming for bidirectional data? +- STM patterns for WebSocket message routing? +- Resource cleanup on connection drop? + +#### **3. Performance Benchmarks** +- What's nginx WebSocket performance? (baseline) +- What's Warp native WebSocket performance? +- Overhead of proxying vs direct serving? +- Can we beat nginx? + +#### **4. Real-World Problems** +Search for: +- "nginx websocket not working" +- "nginx streaming chunked transfer" +- "websocket proxy buffering issues" +- "socket.io nginx configuration problems" + +Learn from what DOESN'T work! + +--- + +## 🏗️ **High-Level Implementation Plan** (Tentative) + +### **Phase 1: Detection & Handshake** +**Goal:** Detect WebSocket upgrade request, proxy handshake + +``` +Request comes in → Check headers (Upgrade: websocket) → + If YES: WebSocket path + If NO: Normal HTTP path (existing code) +``` + +**Key:** Don't break existing HTTP proxying! + +### **Phase 2: Connection Upgrade** +**Goal:** Establish proxy connection to backend WebSocket + +``` +Client ←→ Proxy ←→ Backend + WebSocket WebSocket +``` + +**Challenge:** Maintain two WebSocket connections, proxy frames bidirectionally + +### **Phase 3: Bidirectional Streaming** +**Goal:** Stream frames in BOTH directions simultaneously + +```haskell +-- Conceptual: +forkIO $ forever $ do + clientFrame <- receiveFromClient + sendToBackend clientFrame + +forkIO $ forever $ do + backendFrame <- receiveFromBackend + sendToClient backendFrame +``` + +**Challenge:** Handle connection drops, timeouts, proper cleanup + +### **Phase 4: Integration with Streaming** +**Goal:** Ensure WebSockets + chunked transfer streaming coexist + +**Test Case:** +- WebSocket on `/api/socket.io/` +- Chunked streaming on `/api/ollama/stream` +- BOTH working simultaneously without conflicts! + +--- + +## ❓ **Open Questions & Decisions** + +### **1. Library Choice** +**Options:** +- A) Use `websockets` library (mature, battle-tested) +- B) Use Warp's native WebSocket support +- C) Roll our own (probably stupid) + +**Need to research:** Which integrates better with our existing WAI app? + +### **2. Frame Handling** +**Options:** +- A) Parse WebSocket frames (inspect, modify, validate) +- B) Proxy frames opaquely (zero-copy, faster, less control) + +**Trade-off:** Performance vs observability/security + +### **3. Connection State** +**How to track WebSocket connections?** +- Add to load balancer stats? +- Separate WebSocket connection pool? +- Integrate with health checking? + +### **4. Timeout Strategy** +**Nginx problem:** Different timeouts for HTTP vs WebSocket vs Streaming + +**Our approach:** +- Configurable per-route? +- Detect connection type and auto-adjust? +- Global defaults with overrides? + +--- + +## 🎯 **Success Criteria** + +**Minimum (Just Working):** +- ✅ Detects WebSocket handshake +- ✅ Proxies upgrade to backend +- ✅ Bidirectional frame proxying +- ✅ Handles connection drops + +**Good (Better than Basic):** +- ✅ Zero-copy frame proxying (performance) +- ✅ Works with TLS (wss://) +- ✅ Integrates with load balancing +- ✅ Health checks don't break WebSockets + +**Excellent (Better than Nginx):** +- ✅ **WebSockets + Streaming work SIMULTANEOUSLY without conflicts** +- ✅ Auto-detection (no special config needed for WebSockets) +- ✅ Better error messages than nginx +- ✅ Observable (metrics on WebSocket connections) +- ✅ Faster than nginx WebSocket proxying + +--- + +## 📋 **Next Steps** + +1. **Read existing research docs** (http2-http3.md, performance-optimization.md) +2. **Deep dive into websockets library** - read source if needed +3. **Find nginx conflict root cause** - search Stack Overflow, GitHub issues +4. **Study WAI WebSocket examples** - how does Warp do it natively? +5. **Benchmark nginx WebSocket performance** - set a target to beat +6. **Design the architecture** - how it integrates with existing Proxy.hs +7. **Write research doc** - `websockets-implementation.md` with full details + +**Then:** Start coding with confidence, knowing we have a solid plan! + +--- + +## 💡 **Insights to Remember** + +- WebSockets are just HTTP upgrades - leverage existing HTTP proxying code +- Bidirectional = two concurrent loops, not sequential +- Connection cleanup is critical - use `bracket` pattern +- Performance matters - socket.io can have 1000s of connections +- The nginx conflict is REAL - don't repeat their mistakes + +**Philosophy:** Research first, code smart, build something genuinely better. diff --git a/PROJECTS/Aenebris/docs/research/config-design.md b/PROJECTS/Aenebris/docs/proxy-haskell-research/config-design.md similarity index 100% rename from PROJECTS/Aenebris/docs/research/config-design.md rename to PROJECTS/Aenebris/docs/proxy-haskell-research/config-design.md diff --git a/PROJECTS/Aenebris/docs/research/ddos-mitigation.md b/PROJECTS/Aenebris/docs/proxy-haskell-research/ddos-mitigation.md similarity index 100% rename from PROJECTS/Aenebris/docs/research/ddos-mitigation.md rename to PROJECTS/Aenebris/docs/proxy-haskell-research/ddos-mitigation.md diff --git a/PROJECTS/Aenebris/docs/research/deployment.md b/PROJECTS/Aenebris/docs/proxy-haskell-research/deployment.md similarity index 100% rename from PROJECTS/Aenebris/docs/research/deployment.md rename to PROJECTS/Aenebris/docs/proxy-haskell-research/deployment.md diff --git a/PROJECTS/Aenebris/docs/research/haskell-networking.md b/PROJECTS/Aenebris/docs/proxy-haskell-research/haskell-networking.md similarity index 100% rename from PROJECTS/Aenebris/docs/research/haskell-networking.md rename to PROJECTS/Aenebris/docs/proxy-haskell-research/haskell-networking.md diff --git a/PROJECTS/Aenebris/docs/research/http2-http3.md b/PROJECTS/Aenebris/docs/proxy-haskell-research/http2-http3.md similarity index 100% rename from PROJECTS/Aenebris/docs/research/http2-http3.md rename to PROJECTS/Aenebris/docs/proxy-haskell-research/http2-http3.md diff --git a/PROJECTS/Aenebris/docs/research/load-balancing.md b/PROJECTS/Aenebris/docs/proxy-haskell-research/load-balancing.md similarity index 100% rename from PROJECTS/Aenebris/docs/research/load-balancing.md rename to PROJECTS/Aenebris/docs/proxy-haskell-research/load-balancing.md diff --git a/PROJECTS/Aenebris/docs/research/ml-bot-detection.md b/PROJECTS/Aenebris/docs/proxy-haskell-research/ml-bot-detection.md similarity index 100% rename from PROJECTS/Aenebris/docs/research/ml-bot-detection.md rename to PROJECTS/Aenebris/docs/proxy-haskell-research/ml-bot-detection.md diff --git a/PROJECTS/Aenebris/docs/research/performance-optimization.md b/PROJECTS/Aenebris/docs/proxy-haskell-research/performance-optimization.md similarity index 100% rename from PROJECTS/Aenebris/docs/research/performance-optimization.md rename to PROJECTS/Aenebris/docs/proxy-haskell-research/performance-optimization.md diff --git a/PROJECTS/Aenebris/docs/research/rate-limiting.md b/PROJECTS/Aenebris/docs/proxy-haskell-research/rate-limiting.md similarity index 100% rename from PROJECTS/Aenebris/docs/research/rate-limiting.md rename to PROJECTS/Aenebris/docs/proxy-haskell-research/rate-limiting.md diff --git a/PROJECTS/Aenebris/docs/research/tls-ssl.md b/PROJECTS/Aenebris/docs/proxy-haskell-research/tls-ssl.md similarity index 100% rename from PROJECTS/Aenebris/docs/research/tls-ssl.md rename to PROJECTS/Aenebris/docs/proxy-haskell-research/tls-ssl.md diff --git a/PROJECTS/Aenebris/docs/research/waf-design.md b/PROJECTS/Aenebris/docs/proxy-haskell-research/waf-design.md similarity index 100% rename from PROJECTS/Aenebris/docs/research/waf-design.md rename to PROJECTS/Aenebris/docs/proxy-haskell-research/waf-design.md diff --git a/PROJECTS/api-security-scanner/.env.example b/PROJECTS/api-security-scanner/.env.example index e9596bf6..66a0a3b7 100644 --- a/PROJECTS/api-security-scanner/.env.example +++ b/PROJECTS/api-security-scanner/.env.example @@ -9,8 +9,6 @@ VERSION="1.0.0" DEBUG=true # Security -# IMPORTANT: Generate a secure random key for production! -# Example: openssl rand -hex 32 SECRET_KEY=your-secret-key-change-this-in-production ALGORITHM=HS256 ACCESS_TOKEN_EXPIRE_MINUTES=1440 @@ -24,14 +22,13 @@ POSTGRES_HOST=localhost POSTGRES_PORT=5432 # Database URL (overridden by docker-compose for container networking) -DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} +DATABASE_URL=postgresql+asyncpg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} # Backend API BACKEND_HOST=0.0.0.0 BACKEND_PORT=8000 # CORS Origins (comma-separated for multiple origins) -# Include both direct Vite dev server (5173) and nginx proxy (80) CORS_ORIGINS=http://localhost:5173,http://localhost:3000,http://localhost,http://localhost:8000 # Frontend API URL diff --git a/PROJECTS/api-security-scanner/.pre-commit-config.yaml b/PROJECTS/api-security-scanner/.pre-commit-config.yaml new file mode 100755 index 00000000..68e6fa88 --- /dev/null +++ b/PROJECTS/api-security-scanner/.pre-commit-config.yaml @@ -0,0 +1,34 @@ +# .pre-commit-config.yaml + +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.14.8 + hooks: + - id: ruff + args: ["backend/"] + always_run: true + + - id: mypy-check + name: MyPy Type Checking + entry: bash -c 'cd backend && mypy .' + language: system + types: [python] + pass_filenames: false + always_run: true + + - id: pylint-check + name: PyLint Code Quality + entry: bash -c 'cd backend && pylint .' + language: system + types: [python] + pass_filenames: false + always_run: true + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: check-toml + - id: debug-statements + - id: end-of-file-fixer + - id: check-yaml + args: [--allow-multiple-documents] diff --git a/PROJECTS/api-security-scanner/backend/config.py b/PROJECTS/api-security-scanner/backend/config.py index e0c1d923..a6c028d4 100644 --- a/PROJECTS/api-security-scanner/backend/config.py +++ b/PROJECTS/api-security-scanner/backend/config.py @@ -14,7 +14,6 @@ class Settings(BaseSettings): All magic numbers and configuration values are defined here to avoid hardcoding throughout the application """ - model_config = SettingsConfigDict( env_file="../.env", env_file_encoding="utf-8", case_sensitive=True ) diff --git a/PROJECTS/encrypted-p2p-chat/backend/app/core/surreal_manager.py b/PROJECTS/encrypted-p2p-chat/backend/app/core/surreal_manager.py index 97d28203..c6dd3716 100644 --- a/PROJECTS/encrypted-p2p-chat/backend/app/core/surreal_manager.py +++ b/PROJECTS/encrypted-p2p-chat/backend/app/core/surreal_manager.py @@ -7,6 +7,7 @@ import asyncio import logging from typing import Any from collections.abc import Callable +from datetime import UTC, datetime from surrealdb import AsyncSurreal @@ -240,8 +241,7 @@ class SurrealDBManager: Add a participant to a room """ await self.ensure_connected() - from datetime import UTC, datetime - + query = """ CREATE room_participants CONTENT { room_id: $room_id,