From 823e36eea8707293cdafdc8fb88ca1e56559a9c0 Mon Sep 17 00:00:00 2001 From: Alpamys Date: Mon, 23 Mar 2026 20:55:22 +0500 Subject: [PATCH] =?UTF-8?q?Add=20Web=20UI=20for=20experiment=20management?= =?UTF-8?q?=20(Phase=208)=20=E2=80=94=20v0.7.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `soup ui` command launches local web interface at http://127.0.0.1:7860 - FastAPI backend with REST API: runs, metrics, config validation, training control, data inspection, templates, system info - Self-contained SPA frontend (Dashboard, New Training, Data Explorer, Model Chat) with Chart.js loss/LR charts - Auto-opens browser on launch (--no-browser to disable) - Config validation via new load_config_from_string() in config/loader.py - 40 new tests (530 total), ruff clean Co-Authored-By: Claude Opus 4.6 (1M context) --- CLAUDE.md | 3 + README.md | 23 + pyproject.toml | 8 +- soup_cli/__init__.py | 2 +- soup_cli/cli.py | 2 + soup_cli/commands/ui.py | 72 +++ soup_cli/config/loader.py | 20 + soup_cli/ui/__init__.py | 1 + soup_cli/ui/app.py | 263 +++++++++++ soup_cli/ui/static/app.js | 608 +++++++++++++++++++++++++ soup_cli/ui/static/index.html | 131 ++++++ soup_cli/ui/static/style.css | 365 +++++++++++++++ tests/test_ui.py | 826 ++++++++++++++++++++++++++++++++++ 13 files changed, 2322 insertions(+), 2 deletions(-) create mode 100644 soup_cli/commands/ui.py create mode 100644 soup_cli/ui/__init__.py create mode 100644 soup_cli/ui/app.py create mode 100644 soup_cli/ui/static/app.js create mode 100644 soup_cli/ui/static/index.html create mode 100644 soup_cli/ui/static/style.css create mode 100644 tests/test_ui.py diff --git a/CLAUDE.md b/CLAUDE.md index 2622f1d..f7584bb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -83,6 +83,8 @@ soup train --config soup.yaml **Quickstart:** `commands/quickstart.py` runs a complete demo — creates 20-example alpaca dataset, TinyLlama config, and trains a LoRA adapter. Supports `--dry-run` to create files only. +**Web UI:** `commands/ui.py` launches a local web interface via `soup ui`. `ui/app.py` creates a FastAPI app with REST API endpoints for experiment management (`/api/runs`, `/api/runs/{id}/metrics`), config validation (`/api/config/validate`), training control (`/api/train/start`, `/api/train/status`, `/api/train/stop`), data inspection (`/api/data/inspect`), system info (`/api/system`), and templates (`/api/templates`). `ui/static/` contains a self-contained SPA (HTML/CSS/JS) with four pages: Dashboard (experiments list, loss charts via Chart.js), New Training (config editor with templates), Data Explorer (browse datasets), and Model Chat (chat with a `soup serve` instance). Config validation uses `config/loader.py`'s `load_config_from_string()`. Requires `pip install 'soup-cli[ui]'`. Auto-opens browser on launch (disable with `--no-browser`). + **Confirmation prompts:** `commands/train.py` and `commands/sweep.py` ask for confirmation before starting. Skip with `--yes` / `-y`. **Version:** `cli.py` `version()` command supports `--full` flag that shows version, Python version, GPU backend, and installed optional extras in one line. @@ -171,3 +173,4 @@ Test suite lives in `tests/`: | `test_unsloth.py` | Unsloth backend config, detection, trainer integration, templates | | `test_vision.py` | Vision modality config, LLaVA/ShareGPT4V formats, loader, trainer, templates | | `test_qat.py` | QAT config, validation, trainer integration, export compatibility | +| `test_ui.py` | Web UI command, FastAPI endpoints, static files, config validation | diff --git a/README.md b/README.md index 52c7fec..feceda6 100644 --- a/README.md +++ b/README.md @@ -533,6 +533,27 @@ soup version --full # → soup v0.4.0 | Python 3.11.5 | CUDA 12.1 | extras: serve, data ``` +## Web UI + +Launch a local web interface to manage experiments, start training, explore data, and chat with models — all from your browser. + +```bash +pip install 'soup-cli[ui]' +soup ui +# → opens http://127.0.0.1:7860 in your browser +``` + +**Pages:** +- **Dashboard** — view all experiment runs, loss charts, system info +- **New Training** — create configs from templates, validate, and start training +- **Data Explorer** — browse and inspect datasets (JSONL, JSON, CSV, Parquet) +- **Model Chat** — chat with a running `soup serve` inference server + +```bash +# Custom port, don't auto-open browser +soup ui --port 8080 --no-browser +``` + ## Error Handling Soup shows friendly error messages by default (2-3 lines with a fix suggestion). For full tracebacks: @@ -659,6 +680,7 @@ soup data generate --prompt "..." --count 100 Generate synthetic data soup runs List training runs soup runs show Run details + loss graph soup runs compare Compare two runs +soup ui [--port 7860] Web UI (experiments, training, data) soup doctor Check environment soup quickstart [--dry-run] Full demo soup version [--full] Show version (--full: system info) @@ -678,6 +700,7 @@ soup --verbose Full traceback on errors | `vision` | `pip install 'soup-cli[vision]'` | Vision/multimodal fine-tuning (Pillow) | | `qat` | `pip install 'soup-cli[qat]'` | Quantization-Aware Training (torchao) | | `fast` | `pip install 'soup-cli[fast]'` | Unsloth backend (2-5x faster, -80% VRAM) | +| `ui` | `pip install 'soup-cli[ui]'` | Web UI + inference server (FastAPI + uvicorn) | | `serve` | `pip install 'soup-cli[serve]'` | Inference server (FastAPI + uvicorn) | | `data` | `pip install 'soup-cli[data]'` | Deduplication (MinHash via datasketch) | | `eval` | `pip install 'soup-cli[eval]'` | Benchmark evaluation (lm-evaluation-harness) | diff --git a/pyproject.toml b/pyproject.toml index c6361d8..2250b6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "soup-cli" -version = "0.6.0" +version = "0.7.0" description = "Fine-tune LLMs in one command. No SSH, no config hell." readme = "README.md" license = "MIT" @@ -58,6 +58,12 @@ Homepage = "https://github.com/MakazhanAlpamys/Soup" Repository = "https://github.com/MakazhanAlpamys/Soup" Issues = "https://github.com/MakazhanAlpamys/Soup/issues" +[tool.hatch.build.targets.wheel] +packages = ["soup_cli"] + +[tool.hatch.build.targets.wheel.force-include] +"soup_cli/ui/static" = "soup_cli/ui/static" + [tool.ruff] target-version = "py39" line-length = 100 diff --git a/soup_cli/__init__.py b/soup_cli/__init__.py index 515f6b5..40b06b8 100644 --- a/soup_cli/__init__.py +++ b/soup_cli/__init__.py @@ -1,3 +1,3 @@ """Soup CLI — Fine-tune LLMs in one command.""" -__version__ = "0.6.0" +__version__ = "0.7.0" diff --git a/soup_cli/cli.py b/soup_cli/cli.py index 216554f..8cf5c8d 100644 --- a/soup_cli/cli.py +++ b/soup_cli/cli.py @@ -20,6 +20,7 @@ from soup_cli.commands import ( serve, sweep, train, + ui, ) from soup_cli.commands import doctor as doctor_cmd from soup_cli.commands import quickstart as quickstart_cmd @@ -54,6 +55,7 @@ app.command()(sweep.sweep) app.command(name="diff")(diff.diff) app.command()(doctor_cmd.doctor) app.command()(quickstart_cmd.quickstart) +app.command()(ui.ui) # Register data generate as a subcommand of data data.app.command(name="generate")(generate.generate) diff --git a/soup_cli/commands/ui.py b/soup_cli/commands/ui.py new file mode 100644 index 0000000..7c4c6b2 --- /dev/null +++ b/soup_cli/commands/ui.py @@ -0,0 +1,72 @@ +"""soup ui — local web interface for managing experiments and training.""" + +import typer +from rich.console import Console +from rich.panel import Panel + +console = Console() + + +def ui( + port: int = typer.Option( + 7860, + "--port", + "-p", + help="Port to serve on", + ), + host: str = typer.Option( + "127.0.0.1", + "--host", + help="Host to bind to", + ), + no_browser: bool = typer.Option( + False, + "--no-browser", + help="Don't open browser automatically", + ), +): + """Launch the Soup Web UI for managing experiments and training.""" + try: + import uvicorn # noqa: F401 + from fastapi import FastAPI # noqa: F401 + except ImportError: + console.print( + "[red]FastAPI/uvicorn not installed.[/]\n" + "Install with: [bold]pip install 'soup-cli[ui]'[/]" + ) + raise typer.Exit(1) + + from soup_cli.ui.app import create_app + + app = create_app() + + url = f"http://{host}:{port}" + + console.print( + Panel( + f"URL: [bold]{url}[/]\n\n" + f"Pages:\n" + f" [bold]Dashboard[/] — View experiments, loss charts, system info\n" + f" [bold]New Training[/] — Create config from templates, start training\n" + f" [bold]Data Explorer[/] — Browse and inspect datasets\n" + f" [bold]Model Chat[/] — Chat with a running inference server\n\n" + f"Press [bold]Ctrl+C[/] to stop.", + title="[bold green]Soup Web UI[/]", + ) + ) + + # Open browser + if not no_browser: + import threading + import webbrowser + + def _open(): + import time + time.sleep(1) + webbrowser.open(url) + + threading.Thread(target=_open, daemon=True).start() + + import uvicorn + + uvicorn.run(app, host=host, port=port, log_level="warning") diff --git a/soup_cli/config/loader.py b/soup_cli/config/loader.py index 31894b9..1ebb592 100644 --- a/soup_cli/config/loader.py +++ b/soup_cli/config/loader.py @@ -29,3 +29,23 @@ def load_config(path: Path) -> SoupConfig: raise SystemExit(1) return config + + +def load_config_from_string(yaml_str: str) -> SoupConfig: + """Parse a YAML string and return validated SoupConfig. + + Unlike load_config(), raises ValueError on errors instead of SystemExit, + making it suitable for API/UI usage. + """ + raw = yaml.safe_load(yaml_str) + if raw is None: + raise ValueError("Config is empty") + + try: + return SoupConfig(**raw) + except ValidationError as exc: + errors = [] + for err in exc.errors(): + loc = " → ".join(str(part) for part in err["loc"]) + errors.append(f"{loc}: {err['msg']}") + raise ValueError("; ".join(errors)) diff --git a/soup_cli/ui/__init__.py b/soup_cli/ui/__init__.py new file mode 100644 index 0000000..fd38216 --- /dev/null +++ b/soup_cli/ui/__init__.py @@ -0,0 +1 @@ +"""Soup Web UI — local web interface for managing experiments and training.""" diff --git a/soup_cli/ui/app.py b/soup_cli/ui/app.py new file mode 100644 index 0000000..282caf4 --- /dev/null +++ b/soup_cli/ui/app.py @@ -0,0 +1,263 @@ +"""FastAPI application for Soup Web UI.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import threading +from pathlib import Path +from typing import Optional + +from fastapi import FastAPI, HTTPException, Query +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import HTMLResponse +from fastapi.staticfiles import StaticFiles +from pydantic import BaseModel as PydanticBaseModel + +STATIC_DIR = Path(__file__).parent / "static" + + +class TrainRequest(PydanticBaseModel): + """Request body for starting a training run.""" + config_yaml: str + config_path: Optional[str] = None + + +class TrainStatus(PydanticBaseModel): + """Current training process status.""" + running: bool + pid: Optional[int] = None + config_path: Optional[str] = None + + +class DataInspectRequest(PydanticBaseModel): + """Request body for data inspection.""" + path: str + limit: int = 50 + + +# Global state for training process +_train_process: Optional[subprocess.Popen] = None +_train_config_path: Optional[str] = None +_train_lock = threading.Lock() + + +def create_app() -> FastAPI: + """Create the Soup Web UI FastAPI application.""" + app = FastAPI(title="Soup Web UI", version="1.0.0") + + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], + ) + + # --- Static files --- + + @app.get("/", response_class=HTMLResponse) + def index(): + index_path = STATIC_DIR / "index.html" + return HTMLResponse(content=index_path.read_text(encoding="utf-8")) + + app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") + + # --- Runs API --- + + @app.get("/api/runs") + def list_runs(limit: int = Query(default=50, ge=1, le=500)): + from soup_cli.experiment.tracker import ExperimentTracker + + tracker = ExperimentTracker() + try: + runs = tracker.list_runs(limit=limit) + return {"runs": runs} + finally: + tracker.close() + + @app.get("/api/runs/{run_id}") + def get_run(run_id: str): + from soup_cli.experiment.tracker import ExperimentTracker + + tracker = ExperimentTracker() + try: + run = tracker.get_run(run_id) + if not run: + raise HTTPException(status_code=404, detail="Run not found") + return run + finally: + tracker.close() + + @app.get("/api/runs/{run_id}/metrics") + def get_run_metrics(run_id: str): + from soup_cli.experiment.tracker import ExperimentTracker + + tracker = ExperimentTracker() + try: + run = tracker.get_run(run_id) + if not run: + raise HTTPException(status_code=404, detail="Run not found") + metrics = tracker.get_metrics(run_id) + return {"run_id": run_id, "metrics": metrics} + finally: + tracker.close() + + @app.delete("/api/runs/{run_id}") + def delete_run(run_id: str): + from soup_cli.experiment.tracker import ExperimentTracker + + tracker = ExperimentTracker() + try: + deleted = tracker.delete_run(run_id) + if not deleted: + raise HTTPException(status_code=404, detail="Run not found") + return {"deleted": True, "run_id": run_id} + finally: + tracker.close() + + @app.get("/api/runs/{run_id}/eval") + def get_run_eval(run_id: str): + from soup_cli.experiment.tracker import ExperimentTracker + + tracker = ExperimentTracker() + try: + results = tracker.get_eval_results(run_id=run_id) + return {"run_id": run_id, "eval_results": results} + finally: + tracker.close() + + # --- GPU / System Info --- + + @app.get("/api/system") + def system_info(): + from soup_cli import __version__ + from soup_cli.utils.gpu import detect_device, get_gpu_info + + device, device_name = detect_device() + gpu_info = get_gpu_info() + return { + "version": __version__, + "device": device, + "device_name": device_name, + "gpu_info": gpu_info, + "python_version": sys.version.split()[0], + } + + # --- Templates --- + + @app.get("/api/templates") + def list_templates(): + from soup_cli.config.schema import TEMPLATES + + return {"templates": {name: yaml_str for name, yaml_str in TEMPLATES.items()}} + + # --- Config Validation --- + + @app.post("/api/config/validate") + def validate_config(body: dict): + from soup_cli.config.loader import load_config_from_string + + yaml_str = body.get("yaml", "") + if not yaml_str: + raise HTTPException(status_code=400, detail="Empty config") + try: + config = load_config_from_string(yaml_str) + return {"valid": True, "config": config.model_dump()} + except Exception as exc: + return {"valid": False, "error": str(exc)} + + # --- Training --- + + @app.post("/api/train/start") + def start_training(req: TrainRequest): + global _train_process, _train_config_path + + with _train_lock: + if _train_process and _train_process.poll() is None: + raise HTTPException( + status_code=409, detail="Training already in progress" + ) + + # Write config to temp file + config_path = req.config_path or os.path.join( + os.getcwd(), ".soup_ui_config.yaml" + ) + with open(config_path, "w", encoding="utf-8") as fh: + fh.write(req.config_yaml) + + _train_config_path = config_path + _train_process = subprocess.Popen( + [sys.executable, "-m", "soup_cli", "train", "--config", config_path, "--yes"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + return {"started": True, "pid": _train_process.pid, "config_path": config_path} + + @app.get("/api/train/status") + def train_status(): + global _train_process + with _train_lock: + if _train_process is None: + return TrainStatus(running=False) + poll = _train_process.poll() + if poll is None: + return TrainStatus( + running=True, + pid=_train_process.pid, + config_path=_train_config_path, + ) + return TrainStatus(running=False, pid=_train_process.pid) + + @app.post("/api/train/stop") + def stop_training(): + global _train_process + with _train_lock: + if _train_process and _train_process.poll() is None: + _train_process.terminate() + return {"stopped": True} + return {"stopped": False, "detail": "No training in progress"} + + # --- Data Inspection --- + + @app.post("/api/data/inspect") + def inspect_data(req: DataInspectRequest): + from soup_cli.data.loader import load_raw_data + + path = Path(req.path) + if not path.exists(): + raise HTTPException(status_code=404, detail=f"File not found: {req.path}") + + try: + raw_data = load_raw_data(path) + except Exception as exc: + raise HTTPException(status_code=400, detail=str(exc)) + + total = len(raw_data) + sample = raw_data[: req.limit] + + # Detect format + from soup_cli.data.formats import detect_format + + fmt = detect_format(raw_data[:5]) if raw_data else "unknown" + + # Basic stats + keys = set() + for entry in sample: + keys.update(entry.keys()) + + return { + "path": str(path), + "total": total, + "format": fmt, + "keys": sorted(keys), + "sample": sample, + } + + # --- Health --- + + @app.get("/api/health") + def health(): + return {"status": "ok"} + + return app diff --git a/soup_cli/ui/static/app.js b/soup_cli/ui/static/app.js new file mode 100644 index 0000000..a567332 --- /dev/null +++ b/soup_cli/ui/static/app.js @@ -0,0 +1,608 @@ +/* Soup Web UI — Frontend Application */ + +const API = ''; // same origin + +// --- State --- +let currentPage = 'dashboard'; +let runsData = []; +let systemInfo = null; +let chatMessages = []; +let chatEndpoint = null; + +// --- Navigation --- +function navigate(page) { + currentPage = page; + document.querySelectorAll('.page').forEach(p => p.classList.remove('active')); + document.querySelectorAll('.nav-item').forEach(n => n.classList.remove('active')); + document.getElementById('page-' + page).classList.add('active'); + document.querySelector(`[data-page="${page}"]`).classList.add('active'); + + if (page === 'dashboard') loadDashboard(); + else if (page === 'training') loadTrainingPage(); + else if (page === 'data') { /* loaded on demand */ } + else if (page === 'chat') loadChatPage(); +} + +// --- API Helpers --- +async function api(path, opts = {}) { + const resp = await fetch(API + path, { + headers: { 'Content-Type': 'application/json' }, + ...opts, + }); + if (!resp.ok) { + const err = await resp.json().catch(() => ({ detail: resp.statusText })); + throw new Error(err.detail || 'API error'); + } + return resp.json(); +} + +function formatDuration(secs) { + if (!secs) return '-'; + if (secs < 60) return `${Math.round(secs)}s`; + if (secs < 3600) return `${Math.floor(secs / 60)}m ${Math.round(secs % 60)}s`; + return `${Math.floor(secs / 3600)}h ${Math.floor((secs % 3600) / 60)}m`; +} + +function formatDate(iso) { + if (!iso) return '-'; + const d = new Date(iso); + return d.toLocaleDateString() + ' ' + d.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }); +} + +function statusBadge(status) { + const map = { + completed: 'badge-success', + failed: 'badge-danger', + running: 'badge-warning', + }; + return `${status}`; +} + +function truncate(str, len = 30) { + if (!str) return '-'; + return str.length > len ? str.substring(0, len) + '...' : str; +} + +// --- Dashboard --- +async function loadDashboard() { + try { + const [runsResp, sysResp] = await Promise.all([ + api('/api/runs?limit=100'), + api('/api/system'), + ]); + runsData = runsResp.runs; + systemInfo = sysResp; + renderDashboard(); + } catch (err) { + document.getElementById('dashboard-content').innerHTML = + `
Error loading dashboard: ${err.message}
`; + } +} + +function renderDashboard() { + const completed = runsData.filter(r => r.status === 'completed'); + const failed = runsData.filter(r => r.status === 'failed'); + const running = runsData.filter(r => r.status === 'running'); + const bestLoss = completed.length + ? Math.min(...completed.map(r => r.final_loss).filter(Boolean)).toFixed(4) + : '-'; + + document.getElementById('dashboard-content').innerHTML = ` +
+
+
${runsData.length}
+
Total Runs
+
+
+
${completed.length}
+
Completed
+
+
+
${running.length}
+
Running
+
+
+
${bestLoss}
+
Best Loss
+
+
+ +
+
System
+
+ Device: ${systemInfo.device_name}  |  + GPU Memory: ${systemInfo.gpu_info.memory_total}  |  + Python: ${systemInfo.python_version}  |  + Soup: v${systemInfo.version} +
+
+ +
+
Recent Runs
+ ${runsData.length === 0 + ? '
No runs yet
Start training with "soup train" or use the New Training page
' + : renderRunsTable(runsData.slice(0, 20)) + } +
+ `; +} + +function renderRunsTable(runs) { + return ` +
+ + + + + + + + + + + + + + + + ${runs.map(r => ` + + + + + + + + + + + + `).join('')} + +
Run IDNameModelTaskStatusLossDurationDate
${r.run_id.substring(0, 20)}...${r.experiment_name || '-'}${truncate(r.base_model)}${r.task || 'sft'}${statusBadge(r.status)}${r.final_loss ? r.final_loss.toFixed(4) : '-'}${formatDuration(r.duration_secs)}${formatDate(r.created_at)} + +
+
+ `; +} + +async function deleteRun(runId) { + if (!confirm('Delete this run and all its metrics?')) return; + try { + await api(`/api/runs/${runId}`, { method: 'DELETE' }); + loadDashboard(); + } catch (err) { + alert('Error: ' + err.message); + } +} + +// --- Run Detail Modal --- +let lossChart = null; + +async function showRunDetail(runId) { + const modal = document.getElementById('run-modal'); + const body = document.getElementById('run-modal-body'); + modal.classList.add('active'); + + body.innerHTML = '
Loading...
'; + + try { + const [run, metricsResp] = await Promise.all([ + api(`/api/runs/${runId}`), + api(`/api/runs/${runId}/metrics`), + ]); + + const config = run.config_json ? JSON.parse(run.config_json) : {}; + const metrics = metricsResp.metrics; + + body.innerHTML = ` +
+
+
Run ID
+
${run.run_id}
+
+
+
Status
+
${statusBadge(run.status)}
+
+
+
Model
+
${run.base_model || '-'}
+
+
+
Task
+
${run.task || 'sft'}
+
+
+
Device
+
${run.device_name || run.device || '-'}
+
+
+
Duration
+
${formatDuration(run.duration_secs)}
+
+
+
Initial Loss
+
${run.initial_loss ? run.initial_loss.toFixed(4) : '-'}
+
+
+
Final Loss
+
${run.final_loss ? run.final_loss.toFixed(4) : '-'}
+
+
+ + ${metrics.length > 0 ? ` +
+
Loss Curve
+
+ +
+
+
+
Learning Rate
+
+ +
+
+ ` : ''} + +
+
Config
+ 
${JSON.stringify(config, null, 2)}
+
+ `; + + if (metrics.length > 0) { + renderCharts(metrics); + } + } catch (err) { + body.innerHTML = `
Error: ${err.message}
`; + } +} + +function renderCharts(metrics) { + const steps = metrics.map(m => m.step); + const losses = metrics.map(m => m.loss); + const lrs = metrics.map(m => m.lr); + + // Loss chart + const lossCtx = document.getElementById('loss-chart'); + if (lossCtx) { + if (lossChart) lossChart.destroy(); + lossChart = new Chart(lossCtx, { + type: 'line', + data: { + labels: steps, + datasets: [{ + label: 'Loss', + data: losses, + borderColor: '#22c55e', + backgroundColor: 'rgba(34, 197, 94, 0.1)', + fill: true, + tension: 0.3, + pointRadius: 0, + borderWidth: 2, + }], + }, + options: { + responsive: true, + maintainAspectRatio: false, + plugins: { legend: { display: false } }, + scales: { + x: { title: { display: true, text: 'Step', color: '#94a3b8' }, ticks: { color: '#94a3b8' }, grid: { color: 'rgba(71,85,105,0.3)' } }, + y: { title: { display: true, text: 'Loss', color: '#94a3b8' }, ticks: { color: '#94a3b8' }, grid: { color: 'rgba(71,85,105,0.3)' } }, + }, + }, + }); + } + + // LR chart + const lrCtx = document.getElementById('lr-chart'); + if (lrCtx) { + new Chart(lrCtx, { + type: 'line', + data: { + labels: steps, + datasets: [{ + label: 'Learning Rate', + data: lrs, + borderColor: '#3b82f6', + backgroundColor: 'rgba(59, 130, 246, 0.1)', + fill: true, + tension: 0.3, + pointRadius: 0, + borderWidth: 2, + }], + }, + options: { + responsive: true, + maintainAspectRatio: false, + plugins: { legend: { display: false } }, + scales: { + x: { title: { display: true, text: 'Step', color: '#94a3b8' }, ticks: { color: '#94a3b8' }, grid: { color: 'rgba(71,85,105,0.3)' } }, + y: { title: { display: true, text: 'LR', color: '#94a3b8' }, ticks: { color: '#94a3b8' }, grid: { color: 'rgba(71,85,105,0.3)' } }, + }, + }, + }); + } +} + +function closeModal() { + document.getElementById('run-modal').classList.remove('active'); +} + +// --- New Training Page --- +async function loadTrainingPage() { + try { + const [templatesResp, statusResp] = await Promise.all([ + api('/api/templates'), + api('/api/train/status'), + ]); + renderTrainingPage(templatesResp.templates, statusResp); + } catch (err) { + document.getElementById('training-content').innerHTML = + `
Error: ${err.message}
`; + } +} + +function renderTrainingPage(templates, status) { + const templateNames = Object.keys(templates); + const editorId = 'config-editor'; + + document.getElementById('training-content').innerHTML = ` +
+
+
+
Template
+
+ +
+
+ +
+
Config (YAML)
+ +
+ +
+ + +
+
+
+ +
+
+
Training Status
+
+ ${status.running + ? `
Running PID: ${status.pid}
+ ` + : '
No training in progress
' + } +
+
+ +
+
Quick Reference
+
+ Tasks: sft, dpo, grpo
+ Backends: transformers, unsloth
+ Modalities: text, vision
+ Quantization: 4bit, 8bit, none
+ Formats: alpaca, sharegpt, chatml, dpo, llava, sharegpt4v
+
+
+
+
+ `; + + // Store templates globally + window._templates = templates; +} + +function loadTemplate() { + const sel = document.getElementById('template-select'); + const editor = document.getElementById('config-editor'); + if (sel.value && window._templates[sel.value]) { + editor.value = window._templates[sel.value]; + } +} + +async function validateConfig() { + const yaml = document.getElementById('config-editor').value; + const statusEl = document.getElementById('config-status'); + try { + const result = await api('/api/config/validate', { + method: 'POST', + body: JSON.stringify({ yaml }), + }); + if (result.valid) { + statusEl.innerHTML = 'Config is valid!'; + } else { + statusEl.innerHTML = `Invalid: ${result.error}`; + } + } catch (err) { + statusEl.innerHTML = `Error: ${err.message}`; + } +} + +async function startTraining() { + const yaml = document.getElementById('config-editor').value; + if (!yaml.trim()) { + alert('Please enter a config'); + return; + } + if (!confirm('Start training with this config?')) return; + + try { + const result = await api('/api/train/start', { + method: 'POST', + body: JSON.stringify({ config_yaml: yaml }), + }); + document.getElementById('config-status').innerHTML = + `Training started! PID: ${result.pid}`; + // Refresh status + loadTrainingPage(); + } catch (err) { + document.getElementById('config-status').innerHTML = + `Error: ${err.message}`; + } +} + +async function stopTraining() { + if (!confirm('Stop the current training run?')) return; + try { + await api('/api/train/stop', { method: 'POST' }); + loadTrainingPage(); + } catch (err) { + alert('Error: ' + err.message); + } +} + +// --- Data Explorer --- +async function inspectData() { + const path = document.getElementById('data-path').value; + if (!path.trim()) { alert('Enter a file path'); return; } + + const limit = parseInt(document.getElementById('data-limit').value) || 50; + const content = document.getElementById('data-content'); + content.innerHTML = '
Loading...
'; + + try { + const result = await api('/api/data/inspect', { + method: 'POST', + body: JSON.stringify({ path, limit }), + }); + renderDataResults(result); + } catch (err) { + content.innerHTML = `
Error: ${err.message}
`; + } +} + +function renderDataResults(data) { + const content = document.getElementById('data-content'); + + content.innerHTML = ` +
+
+
${data.total}
+
Total Entries
+
+
+
${data.format}
+
Detected Format
+
+
+
${data.keys.length}
+
Fields
+
+
+ +
+
Fields: ${data.keys.join(', ')}
+
+ +
+
Sample Data (${data.sample.length} of ${data.total})
+ ${data.sample.map((entry, idx) => ` +
+
#${idx + 1}
+ ${Object.entries(entry).map(([key, val]) => ` +
+ ${key}: + ${typeof val === 'object' ? JSON.stringify(val).substring(0, 200) : String(val).substring(0, 200)} +
+ `).join('')} +
+ `).join('')} +
+ `; +} + +// --- Model Chat --- +function loadChatPage() { + // Just ensure the page renders with current messages + renderChatMessages(); +} + +function renderChatMessages() { + const container = document.getElementById('chat-messages'); + if (!container) return; + + if (chatMessages.length === 0) { + container.innerHTML = ` +
+
No messages yet
+
Enter a server URL and start chatting
+
+ `; + return; + } + + container.innerHTML = chatMessages.map(msg => ` +
+
${msg.role}
+
${escapeHtml(msg.content)}
+
+ `).join(''); + + container.scrollTop = container.scrollHeight; +} + +function escapeHtml(text) { + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; +} + +async function sendChatMessage() { + const input = document.getElementById('chat-input'); + const serverUrl = document.getElementById('chat-server').value.trim(); + const msg = input.value.trim(); + if (!msg) return; + if (!serverUrl) { alert('Enter a server URL (e.g., http://localhost:8000)'); return; } + + chatMessages.push({ role: 'user', content: msg }); + input.value = ''; + renderChatMessages(); + + try { + const resp = await fetch(serverUrl + '/v1/chat/completions', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + messages: chatMessages.map(m => ({ role: m.role, content: m.content })), + max_tokens: 512, + temperature: 0.7, + }), + }); + const data = await resp.json(); + const reply = data.choices[0].message.content; + chatMessages.push({ role: 'assistant', content: reply }); + renderChatMessages(); + } catch (err) { + chatMessages.push({ role: 'assistant', content: `[Error: ${err.message}]` }); + renderChatMessages(); + } +} + +function clearChat() { + chatMessages = []; + renderChatMessages(); +} + +function handleChatKey(event) { + if (event.key === 'Enter' && !event.shiftKey) { + event.preventDefault(); + sendChatMessage(); + } +} + +// --- Init --- +document.addEventListener('DOMContentLoaded', () => { + navigate('dashboard'); +}); diff --git a/soup_cli/ui/static/index.html b/soup_cli/ui/static/index.html new file mode 100644 index 0000000..da9db3e --- /dev/null +++ b/soup_cli/ui/static/index.html @@ -0,0 +1,131 @@ + + + + + + Soup — Web UI + + + + +
+ + + + +
+ + +
+
+

Dashboard

+ +
+
+
Loading...
+
+
+ + +
+
+

New Training

+
+
+
Loading...
+
+
+ + +
+
+

Data Explorer

+
+
+
Load Dataset
+
+
+ + +
+
+ + +
+ +
+
+
+
+
Enter a file path to explore your dataset
+
Supports JSONL, JSON, CSV, and Parquet files
+
+
+
+ + +
+
+

Model Chat

+ +
+
+
+
+ + +
+
+
+
+
+
+
No messages yet
+
Start a soup serve server and start chatting
+
+
+
+ + +
+
+
+ +
+
+ + + + + + + diff --git a/soup_cli/ui/static/style.css b/soup_cli/ui/static/style.css new file mode 100644 index 0000000..03e1a45 --- /dev/null +++ b/soup_cli/ui/static/style.css @@ -0,0 +1,365 @@ +/* Soup Web UI — Custom styles */ +:root { + --bg: #0f172a; + --bg-card: #1e293b; + --bg-input: #334155; + --border: #475569; + --text: #f1f5f9; + --text-dim: #94a3b8; + --accent: #22c55e; + --accent-hover: #16a34a; + --danger: #ef4444; + --warning: #f59e0b; + --info: #3b82f6; +} + +* { margin: 0; padding: 0; box-sizing: border-box; } + +body { + font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; + background: var(--bg); + color: var(--text); + min-height: 100vh; +} + +/* Layout */ +.app { display: flex; min-height: 100vh; } + +.sidebar { + width: 220px; + background: var(--bg-card); + border-right: 1px solid var(--border); + padding: 1.5rem 0; + flex-shrink: 0; + display: flex; + flex-direction: column; +} + +.sidebar-logo { + padding: 0 1.5rem 1.5rem; + font-size: 1.5rem; + font-weight: 700; + color: var(--accent); + border-bottom: 1px solid var(--border); + margin-bottom: 1rem; +} + +.sidebar-nav { flex: 1; } + +.nav-item { + display: flex; + align-items: center; + gap: 0.75rem; + padding: 0.75rem 1.5rem; + color: var(--text-dim); + cursor: pointer; + transition: all 0.15s; + font-size: 0.9rem; + border: none; + background: none; + width: 100%; + text-align: left; +} + +.nav-item:hover { background: var(--bg-input); color: var(--text); } +.nav-item.active { color: var(--accent); background: rgba(34, 197, 94, 0.1); border-right: 3px solid var(--accent); } + +.sidebar-footer { + padding: 1rem 1.5rem; + border-top: 1px solid var(--border); + font-size: 0.8rem; + color: var(--text-dim); +} + +.main { flex: 1; padding: 2rem; overflow-y: auto; } + +.page { display: none; } +.page.active { display: block; } + +/* Page header */ +.page-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 1.5rem; +} + +.page-title { font-size: 1.5rem; font-weight: 600; } + +/* Cards */ +.card { + background: var(--bg-card); + border: 1px solid var(--border); + border-radius: 0.75rem; + padding: 1.25rem; + margin-bottom: 1rem; +} + +.card-title { + font-size: 0.85rem; + font-weight: 600; + color: var(--text-dim); + text-transform: uppercase; + letter-spacing: 0.05em; + margin-bottom: 0.75rem; +} + +/* Stats row */ +.stats-row { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); + gap: 1rem; + margin-bottom: 1.5rem; +} + +.stat-card { text-align: center; } +.stat-value { font-size: 2rem; font-weight: 700; color: var(--accent); } +.stat-label { font-size: 0.85rem; color: var(--text-dim); margin-top: 0.25rem; } + +/* Tables */ +.table-wrap { overflow-x: auto; } + +table { + width: 100%; + border-collapse: collapse; + font-size: 0.875rem; +} + +th { + text-align: left; + padding: 0.75rem 1rem; + color: var(--text-dim); + font-weight: 600; + border-bottom: 1px solid var(--border); + font-size: 0.8rem; + text-transform: uppercase; + letter-spacing: 0.05em; +} + +td { + padding: 0.75rem 1rem; + border-bottom: 1px solid rgba(71, 85, 105, 0.3); +} + +tr:hover td { background: rgba(71, 85, 105, 0.15); } + +/* Buttons */ +.btn { + display: inline-flex; + align-items: center; + gap: 0.5rem; + padding: 0.5rem 1rem; + border-radius: 0.5rem; + font-size: 0.875rem; + font-weight: 500; + cursor: pointer; + border: 1px solid var(--border); + background: var(--bg-input); + color: var(--text); + transition: all 0.15s; +} + +.btn:hover { border-color: var(--text-dim); } +.btn-primary { background: var(--accent); color: #000; border-color: var(--accent); } +.btn-primary:hover { background: var(--accent-hover); } +.btn-danger { background: var(--danger); color: #fff; border-color: var(--danger); } +.btn-danger:hover { opacity: 0.9; } +.btn-sm { padding: 0.35rem 0.75rem; font-size: 0.8rem; } + +/* Badges */ +.badge { + display: inline-block; + padding: 0.2rem 0.6rem; + border-radius: 9999px; + font-size: 0.75rem; + font-weight: 600; +} + +.badge-success { background: rgba(34, 197, 94, 0.2); color: var(--accent); } +.badge-danger { background: rgba(239, 68, 68, 0.2); color: var(--danger); } +.badge-warning { background: rgba(245, 158, 11, 0.2); color: var(--warning); } +.badge-info { background: rgba(59, 130, 246, 0.2); color: var(--info); } + +/* Forms */ +.form-group { margin-bottom: 1rem; } + +.form-label { + display: block; + font-size: 0.85rem; + font-weight: 500; + color: var(--text-dim); + margin-bottom: 0.35rem; +} + +input[type="text"], input[type="number"], select, textarea { + width: 100%; + padding: 0.6rem 0.75rem; + background: var(--bg-input); + border: 1px solid var(--border); + border-radius: 0.5rem; + color: var(--text); + font-size: 0.875rem; + font-family: inherit; +} + +input:focus, select:focus, textarea:focus { + outline: none; + border-color: var(--accent); +} + +textarea { + resize: vertical; + font-family: 'JetBrains Mono', 'Fira Code', monospace; + font-size: 0.8rem; + line-height: 1.5; +} + +/* Chart container */ +.chart-container { + position: relative; + height: 300px; + margin-top: 1rem; +} + +/* Chat UI */ +.chat-container { + display: flex; + flex-direction: column; + height: calc(100vh - 180px); +} + +.chat-messages { + flex: 1; + overflow-y: auto; + padding: 1rem 0; +} + +.chat-msg { + display: flex; + gap: 0.75rem; + margin-bottom: 1rem; + padding: 0.75rem; + border-radius: 0.5rem; +} + +.chat-msg.user { background: rgba(59, 130, 246, 0.1); } +.chat-msg.assistant { background: rgba(34, 197, 94, 0.1); } + +.chat-msg-role { + font-size: 0.75rem; + font-weight: 700; + text-transform: uppercase; + min-width: 70px; + color: var(--text-dim); +} + +.chat-msg.user .chat-msg-role { color: var(--info); } +.chat-msg.assistant .chat-msg-role { color: var(--accent); } + +.chat-msg-content { + flex: 1; + white-space: pre-wrap; + line-height: 1.5; +} + +.chat-input-area { + display: flex; + gap: 0.75rem; + padding-top: 1rem; + border-top: 1px solid var(--border); +} + +.chat-input-area input { + flex: 1; +} + +/* Grid */ +.grid-2 { display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; } +.grid-3 { display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 1rem; } + +/* Run detail modal */ +.modal-overlay { + display: none; + position: fixed; + inset: 0; + background: rgba(0, 0, 0, 0.6); + z-index: 100; + justify-content: center; + align-items: flex-start; + padding-top: 5vh; +} + +.modal-overlay.active { display: flex; } + +.modal { + background: var(--bg-card); + border: 1px solid var(--border); + border-radius: 0.75rem; + width: 90%; + max-width: 900px; + max-height: 85vh; + overflow-y: auto; + padding: 1.5rem; +} + +.modal-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 1rem; +} + +.modal-close { + background: none; + border: none; + color: var(--text-dim); + font-size: 1.5rem; + cursor: pointer; +} + +.modal-close:hover { color: var(--text); } + +/* Data explorer */ +.data-entry { + background: var(--bg-input); + border-radius: 0.5rem; + padding: 1rem; + margin-bottom: 0.75rem; + font-size: 0.85rem; +} + +.data-entry-field { + margin-bottom: 0.5rem; +} + +.data-entry-key { + font-weight: 600; + color: var(--accent); + margin-right: 0.5rem; +} + +/* Empty state */ +.empty-state { + text-align: center; + padding: 3rem; + color: var(--text-dim); +} + +.empty-state-icon { font-size: 3rem; margin-bottom: 1rem; } +.empty-state-text { font-size: 1.1rem; margin-bottom: 0.5rem; } +.empty-state-hint { font-size: 0.85rem; } + +/* Scrollbar */ +::-webkit-scrollbar { width: 6px; } +::-webkit-scrollbar-track { background: transparent; } +::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; } +::-webkit-scrollbar-thumb:hover { background: var(--text-dim); } + +/* Responsive */ +@media (max-width: 768px) { + .sidebar { width: 60px; } + .sidebar-logo { font-size: 1rem; padding: 0 0.5rem 1rem; text-align: center; } + .nav-item span { display: none; } + .nav-item { justify-content: center; padding: 0.75rem; } + .grid-2, .grid-3 { grid-template-columns: 1fr; } +} diff --git a/tests/test_ui.py b/tests/test_ui.py new file mode 100644 index 0000000..c09569f --- /dev/null +++ b/tests/test_ui.py @@ -0,0 +1,826 @@ +"""Tests for soup ui — Web UI command and API endpoints.""" + +import json +import os +from unittest.mock import MagicMock, patch + +import pytest + + +class TestUICommand: + """Test the soup ui CLI command.""" + + def test_ui_command_registered(self): + """soup ui should be a registered command.""" + from typer.testing import CliRunner + + from soup_cli.cli import app + + runner = CliRunner() + result = runner.invoke(app, ["ui", "--help"]) + assert result.exit_code == 0 + assert "web ui" in result.output.lower() or "experiments" in result.output.lower() + + def test_ui_command_fastapi_import_error(self): + """soup ui should fail gracefully if FastAPI not installed.""" + from typer.testing import CliRunner + + from soup_cli.cli import app + + runner = CliRunner() + with patch.dict("sys.modules", {"fastapi": None, "uvicorn": None}): + result = runner.invoke(app, ["ui"]) + assert result.exit_code != 0 + + def test_ui_command_options(self): + """soup ui should accept --port, --host, --no-browser options.""" + from typer.testing import CliRunner + + from soup_cli.cli import app + + runner = CliRunner() + result = runner.invoke(app, ["ui", "--help"]) + assert "--port" in result.output + assert "--host" in result.output + assert "--no-browser" in result.output + + +class TestCreateApp: + """Test the FastAPI app creation.""" + + def test_create_app_returns_fastapi_instance(self): + """create_app should return a FastAPI app.""" + try: + from fastapi import FastAPI + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + app = create_app() + assert isinstance(app, FastAPI) + + def test_app_has_required_routes(self): + """App should have all required API routes.""" + try: + import fastapi # noqa: F401 + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + app = create_app() + routes = [route.path for route in app.routes] + + assert "/" in routes + assert "/api/health" in routes + assert "/api/runs" in routes + assert "/api/runs/{run_id}" in routes + assert "/api/runs/{run_id}/metrics" in routes + assert "/api/runs/{run_id}/eval" in routes + assert "/api/system" in routes + assert "/api/templates" in routes + assert "/api/config/validate" in routes + assert "/api/train/start" in routes + assert "/api/train/status" in routes + assert "/api/train/stop" in routes + assert "/api/data/inspect" in routes + + +class TestHealthEndpoint: + """Test the /api/health endpoint.""" + + def test_health_returns_ok(self): + """Health endpoint should return ok status.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.get("/api/health") + assert response.status_code == 200 + assert response.json()["status"] == "ok" + + +class TestIndexEndpoint: + """Test the index page serving.""" + + def test_index_returns_html(self): + """Root should serve HTML page.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.get("/") + assert response.status_code == 200 + assert "text/html" in response.headers["content-type"] + assert "Soup" in response.text + + def test_index_contains_pages(self): + """Index HTML should contain all page sections.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.get("/") + html = response.text + assert "page-dashboard" in html + assert "page-training" in html + assert "page-data" in html + assert "page-chat" in html + + +class TestSystemEndpoint: + """Test the /api/system endpoint.""" + + def test_system_info(self): + """System endpoint should return version and device info.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.get("/api/system") + assert response.status_code == 200 + data = response.json() + assert "version" in data + assert "device" in data + assert "device_name" in data + assert "gpu_info" in data + assert "python_version" in data + + +class TestTemplatesEndpoint: + """Test the /api/templates endpoint.""" + + def test_list_templates(self): + """Templates endpoint should return all built-in templates.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.get("/api/templates") + assert response.status_code == 200 + data = response.json() + templates = data["templates"] + assert "chat" in templates + assert "code" in templates + assert "reasoning" in templates + assert "vision" in templates + assert "medical" in templates + + def test_templates_contain_yaml(self): + """Each template should contain valid YAML content.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.get("/api/templates") + templates = response.json()["templates"] + for name, yaml_str in templates.items(): + assert "base:" in yaml_str, f"Template {name} missing 'base:'" + assert "data:" in yaml_str, f"Template {name} missing 'data:'" + + +class TestConfigValidation: + """Test the /api/config/validate endpoint.""" + + def test_valid_config(self): + """Should validate a correct config.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + yaml_str = """ +base: meta-llama/Llama-3.1-8B +data: + train: ./data/train.jsonl + format: alpaca +""" + response = client.post( + "/api/config/validate", + json={"yaml": yaml_str}, + ) + assert response.status_code == 200 + data = response.json() + assert data["valid"] is True + assert "config" in data + assert data["config"]["base"] == "meta-llama/Llama-3.1-8B" + + def test_invalid_config(self): + """Should report errors for invalid config.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.post( + "/api/config/validate", + json={"yaml": "invalid: true"}, + ) + assert response.status_code == 200 + data = response.json() + assert data["valid"] is False + assert "error" in data + + def test_empty_config(self): + """Should reject empty config.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.post( + "/api/config/validate", + json={"yaml": ""}, + ) + assert response.status_code == 400 + + +class TestRunsEndpoint: + """Test the /api/runs endpoints.""" + + def test_list_runs_empty(self, tmp_path): + """Should return empty list when no runs exist.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + db_path = tmp_path / "test.db" + with patch.dict(os.environ, {"SOUP_DB_PATH": str(db_path)}): + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.get("/api/runs") + assert response.status_code == 200 + assert response.json()["runs"] == [] + + def test_list_runs_with_data(self, tmp_path): + """Should return runs from the database.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + db_path = tmp_path / "test.db" + with patch.dict(os.environ, {"SOUP_DB_PATH": str(db_path)}): + from soup_cli.experiment.tracker import ExperimentTracker + from soup_cli.ui.app import create_app + + # Create a run + tracker = ExperimentTracker(db_path=db_path) + run_id = tracker.start_run( + config_dict={"base": "test-model", "task": "sft"}, + device="cpu", + device_name="CPU", + gpu_info={"memory_total": "N/A"}, + ) + tracker.close() + + client = TestClient(create_app()) + response = client.get("/api/runs") + assert response.status_code == 200 + runs = response.json()["runs"] + assert len(runs) == 1 + assert runs[0]["run_id"] == run_id + + def test_get_run_detail(self, tmp_path): + """Should return run details by ID.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + db_path = tmp_path / "test.db" + with patch.dict(os.environ, {"SOUP_DB_PATH": str(db_path)}): + from soup_cli.experiment.tracker import ExperimentTracker + from soup_cli.ui.app import create_app + + tracker = ExperimentTracker(db_path=db_path) + run_id = tracker.start_run( + config_dict={"base": "llama", "task": "sft"}, + device="cpu", + device_name="CPU", + gpu_info={"memory_total": "N/A"}, + ) + tracker.close() + + client = TestClient(create_app()) + response = client.get(f"/api/runs/{run_id}") + assert response.status_code == 200 + data = response.json() + assert data["run_id"] == run_id + assert data["base_model"] == "llama" + + def test_get_run_not_found(self, tmp_path): + """Should return 404 for nonexistent run.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + db_path = tmp_path / "test.db" + with patch.dict(os.environ, {"SOUP_DB_PATH": str(db_path)}): + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.get("/api/runs/nonexistent_run_id") + assert response.status_code == 404 + + def test_delete_run(self, tmp_path): + """Should delete a run and return success.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + db_path = tmp_path / "test.db" + with patch.dict(os.environ, {"SOUP_DB_PATH": str(db_path)}): + from soup_cli.experiment.tracker import ExperimentTracker + from soup_cli.ui.app import create_app + + tracker = ExperimentTracker(db_path=db_path) + run_id = tracker.start_run( + config_dict={"base": "test"}, + device="cpu", + device_name="CPU", + gpu_info={"memory_total": "N/A"}, + ) + tracker.close() + + client = TestClient(create_app()) + response = client.delete(f"/api/runs/{run_id}") + assert response.status_code == 200 + assert response.json()["deleted"] is True + + # Verify deleted + response = client.get(f"/api/runs/{run_id}") + assert response.status_code == 404 + + def test_delete_run_not_found(self, tmp_path): + """Should return 404 when deleting nonexistent run.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + db_path = tmp_path / "test.db" + with patch.dict(os.environ, {"SOUP_DB_PATH": str(db_path)}): + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.delete("/api/runs/nonexistent") + assert response.status_code == 404 + + +class TestRunMetrics: + """Test the /api/runs/{id}/metrics endpoint.""" + + def test_get_metrics(self, tmp_path): + """Should return metrics for a run.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + db_path = tmp_path / "test.db" + with patch.dict(os.environ, {"SOUP_DB_PATH": str(db_path)}): + from soup_cli.experiment.tracker import ExperimentTracker + from soup_cli.ui.app import create_app + + tracker = ExperimentTracker(db_path=db_path) + run_id = tracker.start_run( + config_dict={"base": "test"}, + device="cpu", + device_name="CPU", + gpu_info={"memory_total": "N/A"}, + ) + tracker.log_metrics(run_id, step=10, loss=2.5, lr=1e-5) + tracker.log_metrics(run_id, step=20, loss=2.0, lr=9e-6) + tracker.log_metrics(run_id, step=30, loss=1.5, lr=8e-6) + tracker.close() + + client = TestClient(create_app()) + response = client.get(f"/api/runs/{run_id}/metrics") + assert response.status_code == 200 + data = response.json() + assert data["run_id"] == run_id + assert len(data["metrics"]) == 3 + assert data["metrics"][0]["step"] == 10 + assert data["metrics"][0]["loss"] == 2.5 + assert data["metrics"][2]["step"] == 30 + + def test_get_metrics_not_found(self, tmp_path): + """Should return 404 for metrics of nonexistent run.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + db_path = tmp_path / "test.db" + with patch.dict(os.environ, {"SOUP_DB_PATH": str(db_path)}): + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.get("/api/runs/nonexistent/metrics") + assert response.status_code == 404 + + +class TestRunEval: + """Test the /api/runs/{id}/eval endpoint.""" + + def test_get_eval_results(self, tmp_path): + """Should return eval results for a run.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + db_path = tmp_path / "test.db" + with patch.dict(os.environ, {"SOUP_DB_PATH": str(db_path)}): + from soup_cli.experiment.tracker import ExperimentTracker + from soup_cli.ui.app import create_app + + tracker = ExperimentTracker(db_path=db_path) + run_id = tracker.start_run( + config_dict={"base": "test"}, + device="cpu", + device_name="CPU", + gpu_info={"memory_total": "N/A"}, + ) + tracker.save_eval_result( + model_path="./output", + benchmark="mmlu", + score=0.75, + details={"subjects": {"math": 0.8}}, + run_id=run_id, + ) + tracker.close() + + client = TestClient(create_app()) + response = client.get(f"/api/runs/{run_id}/eval") + assert response.status_code == 200 + data = response.json() + assert len(data["eval_results"]) == 1 + assert data["eval_results"][0]["benchmark"] == "mmlu" + assert data["eval_results"][0]["score"] == 0.75 + + +class TestDataInspect: + """Test the /api/data/inspect endpoint.""" + + def test_inspect_jsonl(self, tmp_path): + """Should inspect a JSONL file and return entries.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + data_file = tmp_path / "train.jsonl" + entries = [ + {"instruction": "Say hi", "input": "", "output": "Hello!"}, + {"instruction": "Say bye", "input": "", "output": "Goodbye!"}, + {"instruction": "Count", "input": "to 3", "output": "1, 2, 3"}, + ] + data_file.write_text("\n".join(json.dumps(e) for e in entries), encoding="utf-8") + + client = TestClient(create_app()) + response = client.post( + "/api/data/inspect", + json={"path": str(data_file), "limit": 10}, + ) + assert response.status_code == 200 + data = response.json() + assert data["total"] == 3 + assert data["format"] == "alpaca" + assert "instruction" in data["keys"] + assert len(data["sample"]) == 3 + + def test_inspect_file_not_found(self): + """Should return 404 for nonexistent file.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.post( + "/api/data/inspect", + json={"path": "/nonexistent/file.jsonl"}, + ) + assert response.status_code == 404 + + def test_inspect_with_limit(self, tmp_path): + """Should respect the limit parameter.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + data_file = tmp_path / "data.jsonl" + entries = [ + {"instruction": f"Task {i}", "input": "", "output": f"Result {i}"} + for i in range(20) + ] + data_file.write_text( + "\n".join(json.dumps(e) for e in entries), encoding="utf-8" + ) + + client = TestClient(create_app()) + response = client.post( + "/api/data/inspect", + json={"path": str(data_file), "limit": 5}, + ) + assert response.status_code == 200 + data = response.json() + assert data["total"] == 20 + assert len(data["sample"]) == 5 + + def test_inspect_json_file(self, tmp_path): + """Should inspect a JSON array file.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + data_file = tmp_path / "data.json" + entries = [ + {"instruction": "Hi", "input": "", "output": "Hello"}, + ] + data_file.write_text(json.dumps(entries), encoding="utf-8") + + client = TestClient(create_app()) + response = client.post( + "/api/data/inspect", + json={"path": str(data_file)}, + ) + assert response.status_code == 200 + assert response.json()["total"] == 1 + + +class TestTrainEndpoints: + """Test the /api/train/* endpoints.""" + + def test_train_status_not_running(self): + """Should report not running when no training in progress.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + import soup_cli.ui.app as ui_app_module + from soup_cli.ui.app import create_app + + # Reset global state + ui_app_module._train_process = None + + client = TestClient(create_app()) + response = client.get("/api/train/status") + assert response.status_code == 200 + assert response.json()["running"] is False + + def test_stop_training_not_running(self): + """Should report no training to stop.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + import soup_cli.ui.app as ui_app_module + from soup_cli.ui.app import create_app + + ui_app_module._train_process = None + + client = TestClient(create_app()) + response = client.post("/api/train/stop") + assert response.status_code == 200 + assert response.json()["stopped"] is False + + def test_start_training(self, tmp_path): + """Should start training subprocess.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + import soup_cli.ui.app as ui_app_module + from soup_cli.ui.app import create_app + + ui_app_module._train_process = None + + mock_popen = MagicMock() + mock_popen.poll.return_value = None + mock_popen.pid = 12345 + + with patch("soup_cli.ui.app.subprocess.Popen", return_value=mock_popen): + with patch("soup_cli.ui.app.os.getcwd", return_value=str(tmp_path)): + client = TestClient(create_app()) + response = client.post( + "/api/train/start", + json={"config_yaml": "base: test\ndata:\n train: ./data.jsonl\n"}, + ) + assert response.status_code == 200 + data = response.json() + assert data["started"] is True + assert data["pid"] == 12345 + + # Cleanup + ui_app_module._train_process = None + + def test_start_training_conflict(self, tmp_path): + """Should reject second training when one is running.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + import soup_cli.ui.app as ui_app_module + from soup_cli.ui.app import create_app + + # Simulate running process + mock_proc = MagicMock() + mock_proc.poll.return_value = None + ui_app_module._train_process = mock_proc + + client = TestClient(create_app()) + response = client.post( + "/api/train/start", + json={"config_yaml": "base: test\ndata:\n train: ./data.jsonl\n"}, + ) + assert response.status_code == 409 + + # Cleanup + ui_app_module._train_process = None + + +class TestConfigLoader: + """Test the load_config_from_string helper.""" + + def test_load_valid_config(self): + """Should parse valid YAML config.""" + from soup_cli.config.loader import load_config_from_string + + config = load_config_from_string(""" +base: meta-llama/Llama-3.1-8B +data: + train: ./data/train.jsonl + format: alpaca +""") + assert config.base == "meta-llama/Llama-3.1-8B" + assert config.data.train == "./data/train.jsonl" + + def test_load_empty_config(self): + """Should raise ValueError for empty config.""" + from soup_cli.config.loader import load_config_from_string + + with pytest.raises(ValueError, match="empty"): + load_config_from_string("") + + def test_load_invalid_config(self): + """Should raise ValueError for invalid config.""" + from soup_cli.config.loader import load_config_from_string + + with pytest.raises(ValueError): + load_config_from_string("invalid: true\nno_base: 1") + + def test_load_config_with_all_fields(self): + """Should parse config with all fields.""" + from soup_cli.config.loader import load_config_from_string + + config = load_config_from_string(""" +base: codellama/CodeLlama-7b-Instruct-hf +task: sft +data: + train: ./data.jsonl + format: sharegpt + val_split: 0.15 + max_length: 4096 +training: + epochs: 5 + lr: 1e-5 + lora: + r: 128 + alpha: 32 +output: ./my_output +""") + assert config.task == "sft" + assert config.data.val_split == 0.15 + assert config.training.epochs == 5 + assert config.training.lora.r == 128 + assert config.output == "./my_output" + + +class TestStaticFiles: + """Test static file serving.""" + + def test_static_css(self): + """Should serve CSS file.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.get("/static/style.css") + assert response.status_code == 200 + assert "text/css" in response.headers["content-type"] + + def test_static_js(self): + """Should serve JavaScript file.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.get("/static/app.js") + assert response.status_code == 200 + assert "javascript" in response.headers["content-type"] + + def test_static_dir_exists(self): + """Static directory should exist with required files.""" + from soup_cli.ui.app import STATIC_DIR + + assert STATIC_DIR.exists() + assert (STATIC_DIR / "index.html").exists() + assert (STATIC_DIR / "style.css").exists() + assert (STATIC_DIR / "app.js").exists() + + +class TestRunsLimitParam: + """Test the limit query parameter for runs.""" + + def test_runs_limit_default(self, tmp_path): + """Should use default limit of 50.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + db_path = tmp_path / "test.db" + with patch.dict(os.environ, {"SOUP_DB_PATH": str(db_path)}): + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.get("/api/runs") + assert response.status_code == 200 + + def test_runs_custom_limit(self, tmp_path): + """Should accept custom limit parameter.""" + try: + from fastapi.testclient import TestClient + except ImportError: + pytest.skip("FastAPI not installed") + + db_path = tmp_path / "test.db" + with patch.dict(os.environ, {"SOUP_DB_PATH": str(db_path)}): + from soup_cli.ui.app import create_app + + client = TestClient(create_app()) + response = client.get("/api/runs?limit=10") + assert response.status_code == 200