From 02d985db5ea7e7de28b68209e4930732e8155897 Mon Sep 17 00:00:00 2001 From: jakeaturner Date: Thu, 4 Jun 2026 06:27:40 +0000 Subject: [PATCH] feat(system): add opt-in automatic updates for the core NOMAD app MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a self-update path for the NOMAD admin/core image that runs without a human in the loop, gated by layered safety checks. Recreation remains the sidecar's job; this only decides *whether* to update now and requests it. An hourly job (AutoUpdateJob) evaluates a side-effect-free decision pipeline (AutoUpdateService.evaluate): - opt-in: disabled by default - in a user-configured local-time window (handles midnight wrap) - an eligible release: same major (major bumps stay manual), strictly newer, past a configurable cool-off, GA only (no drafts/prereleases), strict semver - pre-flight: sidecar present, no in-flight system update / downloads / app installs, and sufficient host disk (estimated from the registry manifest) When all pass, it drives SystemUpdateService.requestUpdate() with a vetted tag. Failure backoff auto-disables after 3 genuine update-request failures; transient release-lookup errors are skips (offline-first appliances are routinely without connectivity). Re-enabling clears the backoff state. Settings UI exposes the toggle, window, and cool-off, with live status (eligible target, in/out of window, last result/error). A `node ace auto-update:dry-run` command exercises the full pipeline — and a deterministic --scenarios suite the pure decision logic — without ever triggering an update. New KVStore keys under autoUpdate.* hold config + last-run state. --- README.md | 35 + admin/app/controllers/settings_controller.ts | 6 +- admin/app/controllers/system_controller.ts | 24 + admin/app/jobs/auto_update_job.ts | 80 +++ admin/app/services/auto_update_service.ts | 636 ++++++++++++++++++ .../services/container_registry_service.ts | 60 ++ admin/app/services/system_service.ts | 6 + admin/app/services/system_update_service.ts | 26 +- admin/app/validators/settings.ts | 30 +- admin/commands/auto_update/dry_run.ts | 302 +++++++++ admin/commands/queue/work.ts | 4 + admin/constants/kv_store.ts | 2 +- admin/inertia/hooks/useAutoUpdateStatus.ts | 11 + admin/inertia/lib/api.ts | 9 +- admin/inertia/pages/settings/update.tsx | 177 +++++ admin/start/routes.ts | 1 + admin/types/kv_store.ts | 9 + admin/types/system.ts | 21 + 18 files changed, 1429 insertions(+), 10 deletions(-) create mode 100644 admin/app/jobs/auto_update_job.ts create mode 100644 admin/app/services/auto_update_service.ts create mode 100644 admin/commands/auto_update/dry_run.ts create mode 100644 admin/inertia/hooks/useAutoUpdateStatus.ts diff --git a/README.md b/README.md index 46941d3..417094f 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,41 @@ For now, we recommend using network-level controls to manage access if you're pl ## Contributing Contributions are welcome and appreciated! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on how to contribute to the project. +### Testing Auto-Updates (Dry Run) + +The Command Center can automatically install **minor/patch** updates of itself during a configurable window, after a cool-off period, and only when pre-flight checks pass (sufficient disk for the new image, no downloads or app installs in progress). Major versions always require a manual update. + +Because exercising this logic with real version bumps is impractical, an Ace command runs the **entire decision pipeline without ever triggering an update**. Run it from the `admin/` directory: + +```bash +# 1) Deterministic scenario suite — no network, DB, or Docker required. +# Proves every branch (major-only, cool-off, prerelease/draft, window wrap, …) +# and exits non-zero on failure, so it's safe to wire into CI. +node ace auto-update:dry-run --scenarios + +# 2) Simulate "what would happen if I were running 1.32.0 right now?" +# against the LIVE GitHub releases feed and real pre-flight checks: +node ace auto-update:dry-run --current=1.32.0 --force-enabled + +# 3) Fully offline simulation with a canned release list and a fixed clock: +node ace auto-update:dry-run --current=1.32.0 --force-enabled \ + --releases-file=./fixtures/releases.json --now=2026-06-04T21:00:00Z \ + --window-start=20:00 --window-end=23:00 --cooloff=72 --skip-preflight +``` + +It prints the resolved decision — current version, whether the clock is inside the window, the eligible target (if any), and pre-flight blockers — ending in a clear verdict such as `WOULD UPDATE → v1.33.2` or `WOULD NOT UPDATE (outside-window): …`. **No real update is ever requested.** + +| Flag | Description | +|------|-------------| +| `--scenarios` | Run the built-in deterministic scenario suite and exit | +| `--current=` | Simulate this currently-running version (e.g. `1.32.0`) | +| `--force-enabled` | Treat auto-update as enabled, ignoring the saved setting | +| `--cooloff=` | Override the cool-off period | +| `--window-start=` / `--window-end=` | Override the update window | +| `--now=` | Simulate the clock at a specific time | +| `--releases-file=` | Use a local JSON releases array instead of fetching GitHub (offline) | +| `--skip-preflight` | Bypass the Docker/disk/queue pre-flight checks | + ## Community & Resources - **Website:** [www.projectnomad.us](https://www.projectnomad.us) - Learn more about the project diff --git a/admin/app/controllers/settings_controller.ts b/admin/app/controllers/settings_controller.ts index 24cb4ce..b4a2058 100644 --- a/admin/app/controllers/settings_controller.ts +++ b/admin/app/controllers/settings_controller.ts @@ -3,7 +3,7 @@ import { BenchmarkService } from '#services/benchmark_service' import { MapService } from '#services/map_service' import { OllamaService } from '#services/ollama_service' import { SystemService } from '#services/system_service' -import { getSettingSchema, updateSettingSchema } from '#validators/settings' +import { getSettingSchema, updateSettingSchema, validateSettingValue } from '#validators/settings' import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' @@ -118,6 +118,10 @@ export default class SettingsController { async updateSetting({ request, response }: HttpContext) { const reqData = await request.validateUsing(updateSettingSchema) + const valueError = validateSettingValue(reqData.key, reqData.value) + if (valueError) { + return response.status(422).send({ success: false, message: valueError }) + } await this.systemService.updateSetting(reqData.key, reqData.value) return response.status(200).send({ success: true, message: 'Setting updated successfully' }) } diff --git a/admin/app/controllers/system_controller.ts b/admin/app/controllers/system_controller.ts index 1fc5a0d..4acaab4 100644 --- a/admin/app/controllers/system_controller.ts +++ b/admin/app/controllers/system_controller.ts @@ -2,6 +2,9 @@ import { DockerService } from '#services/docker_service'; import { SystemService } from '#services/system_service' import { SystemUpdateService } from '#services/system_update_service' import { ContainerRegistryService } from '#services/container_registry_service' +import { AutoUpdateService } from '#services/auto_update_service' +import { DownloadService } from '#services/download_service' +import { QueueService } from '#services/queue_service' import { CheckServiceUpdatesJob } from '#jobs/check_service_updates_job' import { affectServiceValidator, @@ -126,6 +129,27 @@ export default class SystemController { response.send({ logs }); } + async getAutoUpdateStatus({ response }: HttpContext) { + // Construct inline reusing already-injected singletons + the QueueService + // singleton (its constructor is private to prevent Redis connection leaks, + // so we must not let the container new a fresh one). + const autoUpdateService = new AutoUpdateService( + this.dockerService, + new DownloadService(QueueService.getInstance()), + this.systemService, + this.systemUpdateService, + this.containerRegistryService + ) + + try { + const status = await autoUpdateService.getStatus() + response.send(status) + } catch (error) { + logger.error({ err: error }, '[SystemController] Failed to get auto-update status') + response.status(500).send({ error: 'Failed to retrieve auto-update status' }) + } + } + async subscribeToReleaseNotes({ request }: HttpContext) { const reqData = await request.validateUsing(subscribeToReleaseNotesValidator); diff --git a/admin/app/jobs/auto_update_job.ts b/admin/app/jobs/auto_update_job.ts new file mode 100644 index 0000000..5131e7a --- /dev/null +++ b/admin/app/jobs/auto_update_job.ts @@ -0,0 +1,80 @@ +import { Job } from 'bullmq' +import { QueueService } from '#services/queue_service' +import { DockerService } from '#services/docker_service' +import { DownloadService } from '#services/download_service' +import { SystemService } from '#services/system_service' +import { SystemUpdateService } from '#services/system_update_service' +import { ContainerRegistryService } from '#services/container_registry_service' +import { AutoUpdateService } from '#services/auto_update_service' +import logger from '@adonisjs/core/services/logger' + +/** + * Hourly job that evaluates whether the NOMAD application should auto-update right + * now and, if so, requests it. All gating (opt-in, window, eligibility, cool-off, + * pre-flight, backoff) lives in {@link AutoUpdateService}; this job is just the + * scheduled trigger. Runs hourly so it can act anywhere inside a user's window + * regardless of the window's length. + */ +export class AutoUpdateJob { + static get queue() { + return 'system' + } + + static get key() { + return 'auto-update' + } + + async handle(_job: Job) { + logger.info('[AutoUpdateJob] Evaluating auto-update...') + + const dockerService = new DockerService() + const autoUpdateService = new AutoUpdateService( + dockerService, + new DownloadService(QueueService.getInstance()), + new SystemService(dockerService), + new SystemUpdateService(), + new ContainerRegistryService() + ) + + const result = await autoUpdateService.attempt() + logger.info(`[AutoUpdateJob] ${result.updated ? 'Updating' : 'No update'}: ${result.reason}`) + return result + } + + static async schedule() { + const queueService = QueueService.getInstance() + const queue = queueService.getQueue(this.queue) + + await queue.upsertJobScheduler( + 'hourly-auto-update', + { pattern: '0 * * * *' }, // Top of every hour; attempt() gates on the window + { + name: this.key, + opts: { + removeOnComplete: { count: 12 }, + removeOnFail: { count: 5 }, + }, + } + ) + + logger.info('[AutoUpdateJob] Auto-update evaluation scheduled with cron: 0 * * * *') + } + + static async dispatch() { + const queueService = QueueService.getInstance() + const queue = queueService.getQueue(this.queue) + + const job = await queue.add( + this.key, + {}, + { + attempts: 1, + removeOnComplete: { count: 12 }, + removeOnFail: { count: 5 }, + } + ) + + logger.info(`[AutoUpdateJob] Dispatched ad-hoc auto-update evaluation job ${job.id}`) + return job + } +} diff --git a/admin/app/services/auto_update_service.ts b/admin/app/services/auto_update_service.ts new file mode 100644 index 0000000..013f209 --- /dev/null +++ b/admin/app/services/auto_update_service.ts @@ -0,0 +1,636 @@ +import { inject } from '@adonisjs/core' +import logger from '@adonisjs/core/services/logger' +import axios from 'axios' +import { DateTime } from 'luxon' +import KVStore from '#models/kv_store' +import Service from '#models/service' +import { DockerService } from '#services/docker_service' +import { DownloadService } from '#services/download_service' +import { SystemService } from '#services/system_service' +import { SystemUpdateService } from '#services/system_update_service' +import { ContainerRegistryService } from '#services/container_registry_service' +import { isNewerVersion, parseMajorVersion } from '../utils/version.js' + +/** Docker image repository for the NOMAD admin/core image (tag applied per-release). */ +const NOMAD_IMAGE_REPO = 'ghcr.io/crosstalk-solutions/project-nomad' +const RELEASES_URL = 'https://api.github.com/repos/Crosstalk-Solutions/project-nomad/releases' + +/** Defaults for user-configurable settings (server-local time window + cool-off). */ +const DEFAULT_WINDOW_START = '02:00' +const DEFAULT_WINDOW_END = '05:00' +const DEFAULT_COOLOFF_HOURS = 72 + +/** Require free space >= imageSize * factor to cover decompressed layers + headroom. */ +const DISK_SAFETY_FACTOR = 2 +/** Conservative fallback when the registry image size can't be determined. */ +const MIN_FREE_BYTES = 5 * 1024 * 1024 * 1024 // 5 GiB +/** Genuine failures before auto-update disables itself to avoid an update loop. */ +const MAX_CONSECUTIVE_FAILURES = 3 + +/** + * Only tags matching strict semver are eligible. Defense-in-depth: the selected + * tag becomes `target_tag`, which the sidecar interpolates into a host-side `sed` + * (install/sidecar-updater/update-watcher.sh) — so a malformed tag must never be + * able to reach it, even though releases come from a trusted repo. + */ +const SEMVER_TAG = /^\d+\.\d+\.\d+$/ + +/** Cache the GitHub releases feed in-process to avoid hammering the API. */ +const RELEASES_CACHE_TTL_MS = 15 * 60 * 1000 +/** Briefly remember a failed fetch so repeated calls don't each block on the timeout. */ +const RELEASES_FAILURE_TTL_MS = 60 * 1000 + +export interface AutoUpdateConfig { + enabled: boolean + windowStart: string + windowEnd: string + cooloffHours: number +} + +export interface EligibleTarget { + version: string + tag: string + publishedAt: string +} + +type BlockerSeverity = 'skip' | 'failure' +export interface Blocker { + reason: string + severity: BlockerSeverity +} + +export interface PreflightResult { + ok: boolean + blockers: Blocker[] +} + +/** Minimal shape of a GitHub release entry we depend on. */ +export interface GithubRelease { + tag_name?: string + published_at?: string + draft?: boolean + prerelease?: boolean +} + +/** + * Inputs that can be injected to exercise the decision pipeline deterministically + * (used by the dry-run command/tests). All are optional; when omitted the real + * settings/clock/GitHub feed/pre-flight are used, exactly as production runs. + */ +export interface EvaluateOverrides { + currentVersion?: string + releases?: GithubRelease[] + now?: DateTime + forceEnabled?: boolean + windowStart?: string + windowEnd?: string + cooloffHours?: number + /** Treat pre-flight as passing without touching Docker/disk/queues. */ + skipPreflight?: boolean + /** Substitute a canned pre-flight result. */ + fakePreflight?: PreflightResult +} + +export type DecisionOutcome = + | 'disabled' + | 'outside-window' + | 'eligibility-error' + | 'no-eligible' + | 'blocked' + | 'ready' + +/** Side-effect-free verdict of the decision pipeline. */ +export interface AutoUpdateDecision { + enabled: boolean + currentVersion: string + config: AutoUpdateConfig + withinWindow: boolean + eligibleTarget: EligibleTarget | null + preflight: PreflightResult | null + outcome: DecisionOutcome + reason: string +} + +export interface AutoUpdateStatus extends AutoUpdateConfig { + currentVersion: string + withinWindow: boolean + eligibleTarget: EligibleTarget | null + lastAttemptAt: string | null + lastResult: string | null + lastError: string | null + consecutiveFailures: number + autoDisabledReason: string | null +} + +/** + * Decision + safety layer for automatic updates of the NOMAD application itself. + * + * It does NOT recreate containers — that remains the sidecar's job. This service + * decides *whether* an update should run right now (opt-in, in-window, an eligible + * minor/patch release exists past its cool-off, pre-flight checks pass) and, if so, + * drives the existing {@link SystemUpdateService.requestUpdate} with an explicit, + * eligibility-vetted image tag. + * + * The window/pre-flight helpers are intentionally generic so a future PR can reuse + * them to auto-update installed apps (driving DockerService.updateContainer instead). + */ +@inject() +export class AutoUpdateService { + constructor( + private dockerService: DockerService, + private downloadService: DownloadService, + private systemService: SystemService, + private systemUpdateService: SystemUpdateService, + private containerRegistryService: ContainerRegistryService + ) {} + + /** In-process cache of the last successful releases fetch (per-process). */ + private static releasesCache: { releases: GithubRelease[]; at: number } | null = null + /** Timestamp of the last failed fetch, for short-lived negative caching. */ + private static releasesFailureAt = 0 + + /** Read user-configurable settings, applying defaults. */ + async getConfig(): Promise { + const [enabled, windowStart, windowEnd, cooloffHours] = await Promise.all([ + KVStore.getValue('autoUpdate.enabled'), + KVStore.getValue('autoUpdate.windowStart'), + KVStore.getValue('autoUpdate.windowEnd'), + KVStore.getValue('autoUpdate.cooloffHours'), + ]) + + const parsedCooloff = Number(cooloffHours) + return { + enabled: enabled ?? false, + windowStart: windowStart || DEFAULT_WINDOW_START, + windowEnd: windowEnd || DEFAULT_WINDOW_END, + // `Number(null) === 0`, so an *unset* value must fall through to the default + // rather than silently resolving to a zero cool-off. An explicit 0 is honored. + cooloffHours: + cooloffHours != null && Number.isFinite(parsedCooloff) && parsedCooloff >= 0 + ? parsedCooloff + : DEFAULT_COOLOFF_HOURS, + } + } + + /** + * Determine whether `now` falls inside the configured update window. The window + * is interpreted in the container's local time (set via the TZ env var). Windows + * that wrap past midnight (start > end, e.g. 22:00-02:00) are handled. + */ + isWithinWindow(config: AutoUpdateConfig, now: DateTime = DateTime.now()): boolean { + const start = this.parseMinutes(config.windowStart) + const end = this.parseMinutes(config.windowEnd) + if (start === null || end === null) return false + + const current = now.hour * 60 + now.minute + if (start === end) return false // zero-length window + if (start < end) { + return current >= start && current < end + } + // Wraps midnight + return current >= start || current < end + } + + private parseMinutes(hhmm: string): number | null { + const match = /^([01]\d|2[0-3]):([0-5]\d)$/.exec(hhmm) + if (!match) return null + return Number(match[1]) * 60 + Number(match[2]) + } + + /** + * Find the newest release that is safe to auto-apply: same major version as the + * running build (major bumps are deliberately left for manual update), strictly + * newer than current, and published at least `cooloffHours` ago. Prereleases and + * drafts are ignored — auto-update never rides early access. + * + * Returns null when nothing qualifies (e.g. only a major bump is newer, or the + * newest eligible release is still inside its cool-off window). + */ + async getEligibleTarget(config: AutoUpdateConfig): Promise { + const releases = await this.fetchReleases() + return this.selectEligibleTarget( + releases, + SystemService.getAppVersion(), + config.cooloffHours, + DateTime.now() + ) + } + + /** + * Fetch the published GitHub releases for the NOMAD repo, cached in-process. + * A successful result is reused for {@link RELEASES_CACHE_TTL_MS} so repeated + * status-page loads don't each hit (and risk rate-limiting) the unauthenticated + * GitHub API. A recent failure is negatively cached for {@link RELEASES_FAILURE_TTL_MS} + * so back-to-back calls while offline don't each block on the request timeout. + */ + async fetchReleases(): Promise { + const now = Date.now() + const cached = AutoUpdateService.releasesCache + if (cached && now - cached.at < RELEASES_CACHE_TTL_MS) { + return cached.releases + } + if (now - AutoUpdateService.releasesFailureAt < RELEASES_FAILURE_TTL_MS) { + throw new Error('GitHub releases fetch recently failed; backing off') + } + + try { + const response = await axios.get(RELEASES_URL, { + headers: { Accept: 'application/vnd.github+json' }, + timeout: 5000, + }) + if (!Array.isArray(response.data)) { + throw new Error('Unexpected response from GitHub releases API') + } + AutoUpdateService.releasesCache = { releases: response.data, at: now } + return response.data + } catch (error) { + AutoUpdateService.releasesFailureAt = now + throw error + } + } + + /** + * Pure selection of the newest auto-applicable release from a release list. + * Extracted so the dry-run command and tests can drive it with fixtures. + * Same major as `currentVersion`, strictly newer, published on/before + * `now - cooloffHours`, prereleases/drafts excluded. Returns null for dev + * builds or when nothing qualifies. + */ + selectEligibleTarget( + releases: GithubRelease[], + currentVersion: string, + cooloffHours: number, + now: DateTime + ): EligibleTarget | null { + if (currentVersion === 'dev' || currentVersion === '0.0.0') { + return null + } + const currentMajor = parseMajorVersion(currentVersion) + const cutoff = now.minus({ hours: cooloffHours }) + + const candidates = releases + .filter((r) => r && !r.draft && !r.prerelease && r.tag_name && r.published_at) + .map((r) => ({ + version: String(r.tag_name).replace(/^v/, '').trim(), + publishedAt: String(r.published_at), + })) + .filter((r) => SEMVER_TAG.test(r.version)) + .filter((r) => parseMajorVersion(r.version) === currentMajor) + .filter((r) => isNewerVersion(r.version, currentVersion)) + .filter((r) => DateTime.fromISO(r.publishedAt) <= cutoff) + .sort((a, b) => (isNewerVersion(a.version, b.version) ? -1 : 1)) + + const best = candidates[0] + if (!best) return null + + return { + version: best.version, + tag: `v${best.version}`, + publishedAt: best.publishedAt, + } + } + + /** + * Pre-flight checks that gate an auto-update. `skip` blockers are transient + * (retry next window, no penalty); `failure` blockers count toward the backoff + * that eventually auto-disables auto-update. + */ + async runPreflight(targetTag: string): Promise { + const blockers: Blocker[] = [] + + // 1. Sidecar must be present to perform the update. + if (!this.systemUpdateService.isSidecarAvailable()) { + blockers.push({ reason: 'Update sidecar is not available', severity: 'failure' }) + } + + // 2. No system update already running. + const updateStatus = this.systemUpdateService.getUpdateStatus() + if (updateStatus && !['idle', 'complete', 'error'].includes(updateStatus.stage)) { + blockers.push({ + reason: `A system update is already in progress (stage: ${updateStatus.stage})`, + severity: 'skip', + }) + } + + // 3. No content/model downloads in progress. + try { + const downloads = await this.downloadService.listDownloadJobs() + const active = downloads.filter( + (d) => !!d.status && ['waiting', 'active', 'delayed'].includes(d.status) + ) + if (active.length > 0) { + blockers.push({ + reason: `${active.length} download(s) in progress`, + severity: 'skip', + }) + } + } catch (error) { + logger.warn(`[AutoUpdateService] Could not check active downloads: ${error.message}`) + } + + // 4. No app (container) install/update in progress. + try { + const installing = await Service.query().whereNot('installation_status', 'idle') + if (installing.length > 0) { + blockers.push({ + reason: `${installing.length} app install/update(s) in progress`, + severity: 'skip', + }) + } + } catch (error) { + logger.warn(`[AutoUpdateService] Could not check app installations: ${error.message}`) + } + + // 5. Sufficient host storage for the new image. + const diskBlocker = await this.checkDiskSpace(targetTag) + if (diskBlocker) blockers.push(diskBlocker) + + return { ok: blockers.length === 0, blockers } + } + + /** Returns a disk blocker if free space is insufficient, otherwise null. */ + private async checkDiskSpace(targetTag: string): Promise { + try { + const hostArch = await this.getHostArch() + const parsed = this.containerRegistryService.parseImageReference( + `${NOMAD_IMAGE_REPO}:${targetTag}` + ) + const imageSize = await this.containerRegistryService.getImageDownloadSize( + parsed, + targetTag, + hostArch + ) + const required = imageSize !== null ? imageSize * DISK_SAFETY_FACTOR : MIN_FREE_BYTES + + const free = await this.getFreeBytes() + if (free === null) { + logger.warn('[AutoUpdateService] Could not determine free disk space; skipping disk check') + return null + } + + if (free < required) { + return { + reason: `Insufficient disk space: ${this.gib(free)} free, ${this.gib(required)} required`, + severity: 'failure', + } + } + return null + } catch (error) { + logger.warn(`[AutoUpdateService] Disk space check failed: ${error.message}`) + return null + } + } + + /** Free bytes on the root filesystem (best-effort, falls back to max available). */ + private async getFreeBytes(): Promise { + const info = await this.systemService.getSystemInfo() + if (!info?.fsSize?.length) return null + const root = info.fsSize.find((f) => f.mount === '/') + if (root) return root.available + return Math.max(...info.fsSize.map((f) => f.available)) + } + + private gib(bytes: number): string { + return `${(bytes / 1024 / 1024 / 1024).toFixed(1)} GiB` + } + + /** Map the Docker daemon's architecture string to OCI naming (amd64/arm64/...). */ + private async getHostArch(): Promise { + try { + const info = await this.dockerService.docker.info() + const arch = info.Architecture || '' + const archMap: Record = { + x86_64: 'amd64', + aarch64: 'arm64', + armv7l: 'arm', + amd64: 'amd64', + arm64: 'arm64', + } + return archMap[arch] || arch.toLowerCase() + } catch { + return 'amd64' + } + } + + /** + * Side-effect-free core of the decision pipeline. Resolves the effective config + * (settings, overridable), checks the window, finds an eligible target, and runs + * pre-flight — returning a verdict WITHOUT requesting an update or mutating any + * persisted state. Both {@link attempt} (production) and {@link dryRun} (testing) + * are built on this so a dry run faithfully reflects what a real run would do. + */ + async evaluate(overrides: EvaluateOverrides = {}): Promise { + const baseConfig = await this.getConfig() + const config: AutoUpdateConfig = { + enabled: overrides.forceEnabled ?? baseConfig.enabled, + windowStart: overrides.windowStart ?? baseConfig.windowStart, + windowEnd: overrides.windowEnd ?? baseConfig.windowEnd, + cooloffHours: overrides.cooloffHours ?? baseConfig.cooloffHours, + } + const now = overrides.now ?? DateTime.now() + const currentVersion = overrides.currentVersion ?? SystemService.getAppVersion() + + const base = { + enabled: config.enabled, + currentVersion, + config, + withinWindow: false, + eligibleTarget: null as EligibleTarget | null, + preflight: null as PreflightResult | null, + } + + if (!config.enabled) { + return { ...base, outcome: 'disabled', reason: 'Auto-update is disabled' } + } + + const withinWindow = this.isWithinWindow(config, now) + if (!withinWindow) { + return { + ...base, + outcome: 'outside-window', + reason: `Outside update window (${config.windowStart}-${config.windowEnd})`, + } + } + + let eligibleTarget: EligibleTarget | null + try { + const releases = overrides.releases ?? (await this.fetchReleases()) + eligibleTarget = this.selectEligibleTarget(releases, currentVersion, config.cooloffHours, now) + } catch (error) { + return { + ...base, + withinWindow, + outcome: 'eligibility-error', + reason: `Failed to determine eligible version: ${error.message}`, + } + } + + if (!eligibleTarget) { + return { + ...base, + withinWindow, + outcome: 'no-eligible', + reason: 'No eligible minor/patch update available (or still in cool-off)', + } + } + + const preflight = overrides.fakePreflight + ? overrides.fakePreflight + : overrides.skipPreflight + ? { ok: true, blockers: [] } + : await this.runPreflight(eligibleTarget.tag) + + if (!preflight.ok) { + const summary = preflight.blockers.map((b) => b.reason).join('; ') + return { + ...base, + withinWindow, + eligibleTarget, + preflight, + outcome: 'blocked', + reason: `Pre-flight blocked: ${summary}`, + } + } + + return { + ...base, + withinWindow, + eligibleTarget, + preflight, + outcome: 'ready', + reason: `Ready to update to ${eligibleTarget.tag}`, + } + } + + /** + * Run the full decision pipeline WITHOUT requesting an update or recording any + * state. Accepts the same injectable overrides as {@link evaluate}, so callers can + * simulate any scenario (a given current version, a canned release list, a fixed + * clock, a forced window) and see exactly what a real run would decide. + */ + async dryRun(overrides: EvaluateOverrides = {}): Promise { + return this.evaluate(overrides) + } + + /** + * The entry point invoked by AutoUpdateJob. Evaluates the decision pipeline and, + * when everything passes, requests the update with the vetted tag — recording the + * outcome to the KVStore (for the UI) and applying failure backoff. + */ + async attempt(): Promise<{ updated: boolean; reason: string }> { + const decision = await this.evaluate() + + switch (decision.outcome) { + case 'disabled': + return { updated: false, reason: decision.reason } + + case 'outside-window': + case 'no-eligible': + // A failed release lookup is transient (offline-first appliances are + // routinely without connectivity) — treat as a skip so it never trips the + // backoff that auto-disables the feature. Only real update-request failures + // (the `ready` case below) count toward MAX_CONSECUTIVE_FAILURES. + case 'eligibility-error': + await this.recordSkip(decision.reason) + return { updated: false, reason: decision.reason } + + case 'blocked': { + const hasFailure = decision.preflight!.blockers.some((b) => b.severity === 'failure') + if (hasFailure) { + await this.recordFailure(decision.reason) + } else { + await this.recordSkip(decision.reason) + } + return { updated: false, reason: decision.reason } + } + + case 'ready': { + const target = decision.eligibleTarget! + const result = await this.systemUpdateService.requestUpdate({ + targetTag: target.tag, + requester: 'auto-update', + }) + + if (result.success) { + await this.recordSuccess(target) + logger.info(`[AutoUpdateService] Auto-update requested: ${target.tag}`) + return { updated: true, reason: `Update requested: ${target.tag}` } + } + + await this.recordFailure(`Update request failed: ${result.message}`) + return { updated: false, reason: result.message } + } + } + } + + // --- Outcome recording ----------------------------------------------------- + + private async recordSuccess(target: EligibleTarget): Promise { + await KVStore.setValue('autoUpdate.lastAttemptAt', DateTime.now().toISO()!) + await KVStore.setValue('autoUpdate.lastResult', `Update requested: ${target.tag}`) + await KVStore.clearValue('autoUpdate.lastError') + await KVStore.setValue('autoUpdate.consecutiveFailures', '0') + } + + private async recordSkip(reason: string): Promise { + await KVStore.setValue('autoUpdate.lastAttemptAt', DateTime.now().toISO()!) + await KVStore.setValue('autoUpdate.lastResult', reason) + logger.info(`[AutoUpdateService] Skipped: ${reason}`) + } + + private async recordFailure(reason: string): Promise { + await KVStore.setValue('autoUpdate.lastAttemptAt', DateTime.now().toISO()!) + await KVStore.setValue('autoUpdate.lastResult', reason) + await KVStore.setValue('autoUpdate.lastError', reason) + + const prior = Number(await KVStore.getValue('autoUpdate.consecutiveFailures')) || 0 + const failures = prior + 1 + await KVStore.setValue('autoUpdate.consecutiveFailures', String(failures)) + logger.error(`[AutoUpdateService] Failure ${failures}/${MAX_CONSECUTIVE_FAILURES}: ${reason}`) + + if (failures >= MAX_CONSECUTIVE_FAILURES) { + await KVStore.setValue('autoUpdate.enabled', false) + await KVStore.setValue( + 'autoUpdate.autoDisabledReason', + `Auto-update disabled after ${failures} consecutive failures. Last error: ${reason}` + ) + logger.error( + `[AutoUpdateService] Auto-update auto-disabled after ${failures} consecutive failures` + ) + } + } + + /** Full state snapshot for the settings UI. */ + async getStatus(): Promise { + const config = await this.getConfig() + const currentVersion = SystemService.getAppVersion() + + let eligibleTarget: EligibleTarget | null = null + try { + eligibleTarget = await this.getEligibleTarget(config) + } catch (error) { + logger.warn(`[AutoUpdateService] getStatus eligibility lookup failed: ${error.message}`) + } + + const [lastAttemptAt, lastResult, lastError, consecutiveFailures, autoDisabledReason] = + await Promise.all([ + KVStore.getValue('autoUpdate.lastAttemptAt'), + KVStore.getValue('autoUpdate.lastResult'), + KVStore.getValue('autoUpdate.lastError'), + KVStore.getValue('autoUpdate.consecutiveFailures'), + KVStore.getValue('autoUpdate.autoDisabledReason'), + ]) + + return { + ...config, + currentVersion, + withinWindow: this.isWithinWindow(config), + eligibleTarget, + lastAttemptAt: lastAttemptAt || null, + lastResult: lastResult || null, + lastError: lastError || null, + consecutiveFailures: Number(consecutiveFailures) || 0, + autoDisabledReason: autoDisabledReason || null, + } + } +} diff --git a/admin/app/services/container_registry_service.ts b/admin/app/services/container_registry_service.ts index 01251fc..afdd39d 100644 --- a/admin/app/services/container_registry_service.ts +++ b/admin/app/services/container_registry_service.ts @@ -200,6 +200,66 @@ export class ContainerRegistryService { } } + /** + * Estimate the compressed download size (in bytes) of an image tag for the + * given host architecture by summing its layer sizes from the manifest. + * + * Resolves a multi-arch manifest list/index down to the platform-specific + * manifest before summing `layers[].size`. Returns null on any failure so + * callers can fall back to a conservative fixed threshold rather than + * silently skipping a disk pre-flight check. + */ + async getImageDownloadSize( + parsed: ParsedImageReference, + tag: string, + hostArch: string + ): Promise { + try { + const token = await this.getToken(parsed.registry, parsed.fullName) + const manifestAccept = [ + 'application/vnd.oci.image.index.v1+json', + 'application/vnd.docker.distribution.manifest.list.v2+json', + 'application/vnd.oci.image.manifest.v1+json', + 'application/vnd.docker.distribution.manifest.v2+json', + ].join(', ') + + const fetchManifest = async (ref: string) => + this.fetchWithRetry(`https://${parsed.registry}/v2/${parsed.fullName}/manifests/${ref}`, { + headers: { Authorization: `Bearer ${token}`, Accept: manifestAccept }, + }) + + const topRes = await fetchManifest(tag) + if (!topRes.ok) return null + + let manifest = (await topRes.json()) as { + mediaType?: string + layers?: Array<{ size?: number }> + manifests?: Array<{ digest?: string; platform?: { architecture?: string } }> + } + + // Multi-arch manifest list/index — resolve to the host-arch child manifest. + if (manifest.manifests?.length) { + const match = + manifest.manifests.find((m) => m.platform?.architecture === hostArch) || + manifest.manifests[0] + if (!match?.digest) return null + + const childRes = await fetchManifest(match.digest) + if (!childRes.ok) return null + manifest = (await childRes.json()) as { layers?: Array<{ size?: number }> } + } + + if (!manifest.layers?.length) return null + + return manifest.layers.reduce((total, layer) => total + (layer.size || 0), 0) + } catch (error) { + logger.warn( + `[ContainerRegistryService] Failed to get image size for ${parsed.fullName}:${tag}: ${error.message}` + ) + return null + } + } + /** * Extract the source repository URL from an image's OCI labels. * Uses the standardized `org.opencontainers.image.source` label. diff --git a/admin/app/services/system_service.ts b/admin/app/services/system_service.ts index 8e27d85..f261496 100644 --- a/admin/app/services/system_service.ts +++ b/admin/app/services/system_service.ts @@ -832,6 +832,12 @@ export class SystemService { if (key === 'ai.assistantCustomName') { invalidateAssistantNameCache() } + // Re-enabling auto-update after a backoff-driven auto-disable clears the + // failure state so it gets a fresh start instead of immediately re-tripping. + if (key === 'autoUpdate.enabled' && (value === true || value === 'true')) { + await KVStore.setValue('autoUpdate.consecutiveFailures', '0') + await KVStore.clearValue('autoUpdate.autoDisabledReason') + } } /** diff --git a/admin/app/services/system_update_service.ts b/admin/app/services/system_update_service.ts index 73f0791..502b9f1 100644 --- a/admin/app/services/system_update_service.ts +++ b/admin/app/services/system_update_service.ts @@ -18,9 +18,18 @@ export class SystemUpdateService { private static LOG_FILE = join(SystemUpdateService.SHARED_DIR, 'update-log') /** - * Requests a system update by creating a request file that the sidecar will detect + * Requests a system update by creating a request file that the sidecar will detect. + * + * @param options.targetTag - Explicit Docker image tag to install (e.g. "v1.33.2"). + * When omitted, falls back to the cached `system.latestVersion` (manual-update + * behavior). Auto-update passes an eligibility-vetted tag here, which may differ + * from `system.latestVersion` when the newest release is a major bump. + * @param options.requester - Identifier recorded in the request file for auditing. */ - async requestUpdate(): Promise<{ success: boolean; message: string }> { + async requestUpdate(options?: { + targetTag?: string + requester?: string + }): Promise<{ success: boolean; message: string }> { try { const currentStatus = this.getUpdateStatus() if (currentStatus && !['idle', 'complete', 'error'].includes(currentStatus.stage)) { @@ -30,13 +39,18 @@ export class SystemUpdateService { } } - // Determine the Docker image tag to install. - const latestVersion = await KVStore.getValue('system.latestVersion') + // Determine the Docker image tag to install. Prefer an explicit caller-supplied + // tag; otherwise use the cached latest version. + let targetTag = options?.targetTag + if (!targetTag) { + const latestVersion = await KVStore.getValue('system.latestVersion') + targetTag = latestVersion ? `v${latestVersion}` : 'latest' + } const requestData = { requested_at: new Date().toISOString(), - requester: 'admin-api', - target_tag: latestVersion ? `v${latestVersion}` : 'latest', + requester: options?.requester ?? 'admin-api', + target_tag: targetTag, } await writeFile(SystemUpdateService.REQUEST_FILE, JSON.stringify(requestData, null, 2)) diff --git a/admin/app/validators/settings.ts b/admin/app/validators/settings.ts index fff9d38..e9067aa 100644 --- a/admin/app/validators/settings.ts +++ b/admin/app/validators/settings.ts @@ -1,5 +1,6 @@ import vine from "@vinejs/vine"; import { SETTINGS_KEYS } from "../../constants/kv_store.js"; +import type { KVStoreKey } from "../../types/kv_store.js"; export const getSettingSchema = vine.compile(vine.object({ key: vine.enum(SETTINGS_KEYS), @@ -8,4 +9,31 @@ export const getSettingSchema = vine.compile(vine.object({ export const updateSettingSchema = vine.compile(vine.object({ key: vine.enum(SETTINGS_KEYS), value: vine.any().optional(), -})) \ No newline at end of file +})) + +const HHMM_PATTERN = /^([01]\d|2[0-3]):[0-5]\d$/ + +/** + * Validate the *value* for keys that have format constraints beyond the generic + * enum/any check (the generic validator only constrains the key). Returns an + * error message string when invalid, or null when the value is acceptable. + */ +export function validateSettingValue(key: KVStoreKey, value: unknown): string | null { + switch (key) { + case 'autoUpdate.windowStart': + case 'autoUpdate.windowEnd': + if (typeof value !== 'string' || !HHMM_PATTERN.test(value)) { + return 'Time window values must be in 24-hour HH:MM format (e.g. "20:00").' + } + return null + case 'autoUpdate.cooloffHours': { + const num = Number(value) + if (!Number.isInteger(num) || num < 0 || num > 8760) { + return 'Cool-off must be a whole number of hours between 0 and 8760.' + } + return null + } + default: + return null + } +} \ No newline at end of file diff --git a/admin/commands/auto_update/dry_run.ts b/admin/commands/auto_update/dry_run.ts new file mode 100644 index 0000000..9590ef2 --- /dev/null +++ b/admin/commands/auto_update/dry_run.ts @@ -0,0 +1,302 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' +import { readFile } from 'node:fs/promises' + +/** + * Exercise the auto-update decision pipeline WITHOUT ever triggering a real update. + * + * # Prove the core logic deterministically (no network/DB/Docker): + * node ace auto-update:dry-run --scenarios + * + * # Simulate "what would happen if I were running 1.32.0 right now" + * # against the live GitHub releases feed and real pre-flight checks: + * node ace auto-update:dry-run --current=1.32.0 --force-enabled + * + * # Fully offline simulation with a canned release list + fixed clock: + * node ace auto-update:dry-run --current=1.32.0 --force-enabled \ + * --releases-file=./fixtures/releases.json --now=2026-06-04T21:00:00Z \ + * --window-start=20:00 --window-end=23:00 --skip-preflight + */ +export default class AutoUpdateDryRun extends BaseCommand { + static commandName = 'auto-update:dry-run' + static description = 'Dry-run the auto-update decision pipeline (never triggers an update)' + + @flags.boolean({ description: 'Run the built-in deterministic scenario suite and exit' }) + declare scenarios: boolean + + @flags.string({ description: 'Simulate this currently-running version (e.g. 1.32.0)' }) + declare current: string + + @flags.boolean({ description: 'Ignore the persisted enabled setting and treat as enabled' }) + declare forceEnabled: boolean + + @flags.string({ description: 'Override cool-off hours' }) + declare cooloff: string + + @flags.string({ description: 'Override window start (HH:MM)' }) + declare windowStart: string + + @flags.string({ description: 'Override window end (HH:MM)' }) + declare windowEnd: string + + @flags.string({ description: 'Simulate the clock at this ISO timestamp' }) + declare now: string + + @flags.string({ description: 'Path to a JSON file with a GitHub releases array (offline)' }) + declare releasesFile: string + + @flags.boolean({ description: 'Bypass Docker/disk/queue pre-flight checks' }) + declare skipPreflight: boolean + + static options: CommandOptions = { + startApp: true, + } + + async run() { + const { DateTime } = await import('luxon') + const { DockerService } = await import('#services/docker_service') + const { DownloadService } = await import('#services/download_service') + const { SystemService } = await import('#services/system_service') + const { SystemUpdateService } = await import('#services/system_update_service') + const { ContainerRegistryService } = await import('#services/container_registry_service') + const { QueueService } = await import('#services/queue_service') + const { AutoUpdateService } = await import('#services/auto_update_service') + + const dockerService = new DockerService() + const svc = new AutoUpdateService( + dockerService, + new DownloadService(QueueService.getInstance()), + new SystemService(dockerService), + new SystemUpdateService(), + new ContainerRegistryService() + ) + + if (this.scenarios) { + const ok = this.runScenarios(svc, DateTime) + if (!ok) { + this.exitCode = 1 + } + return + } + + // --- Live / simulated single dry run ------------------------------------ + const overrides: Record = {} + if (this.current) overrides.currentVersion = this.current + if (this.forceEnabled) overrides.forceEnabled = true + if (this.cooloff) overrides.cooloffHours = Number(this.cooloff) + if (this.windowStart) overrides.windowStart = this.windowStart + if (this.windowEnd) overrides.windowEnd = this.windowEnd + if (this.skipPreflight) overrides.skipPreflight = true + if (this.now) overrides.now = DateTime.fromISO(this.now) + if (this.releasesFile) { + const raw = await readFile(this.releasesFile, 'utf-8') + overrides.releases = JSON.parse(raw) + } + + this.logger.info('Running auto-update dry run (no update will be triggered)...') + const decision = await svc.dryRun(overrides) + + this.logger.log('') + this.logger.log(` Current version : ${decision.currentVersion}`) + this.logger.log(` Enabled : ${decision.enabled}`) + this.logger.log( + ` Window : ${decision.config.windowStart}-${decision.config.windowEnd} ` + + `(currently ${decision.withinWindow ? 'inside' : 'outside'})` + ) + this.logger.log(` Cool-off hours : ${decision.config.cooloffHours}`) + this.logger.log( + ` Eligible target : ${decision.eligibleTarget ? decision.eligibleTarget.tag + ' (published ' + decision.eligibleTarget.publishedAt + ')' : '—'}` + ) + if (decision.preflight) { + if (decision.preflight.ok) { + this.logger.log(` Pre-flight : ok`) + } else { + this.logger.log(` Pre-flight : BLOCKED`) + for (const b of decision.preflight.blockers) { + this.logger.log(` - [${b.severity}] ${b.reason}`) + } + } + } else { + this.logger.log(` Pre-flight : (not reached)`) + } + this.logger.log('') + + const verdict = + decision.outcome === 'ready' + ? `WOULD UPDATE → ${decision.eligibleTarget!.tag}` + : `WOULD NOT UPDATE (${decision.outcome}): ${decision.reason}` + if (decision.outcome === 'ready') { + this.logger.success(verdict) + } else { + this.logger.info(verdict) + } + } + + /** + * Deterministic acceptance suite over the pure decision helpers — no network, + * DB, or Docker. Proves every branch reviewers care about. + */ + private runScenarios(svc: any, DateTime: any): boolean { + const NOW = '2026-06-04T12:00:00Z' + const now = DateTime.fromISO(NOW) + const daysAgo = (d: number) => now.minus({ days: d }).toISO() + const hoursAgo = (h: number) => now.minus({ hours: h }).toISO() + const rel = (tag: string, published: string, extra: object = {}) => ({ + tag_name: tag, + published_at: published, + ...extra, + }) + + type EligCase = { + name: string + releases: any[] + current: string + cooloff: number + expect: string | null + } + + const eligibility: EligCase[] = [ + { + name: 'only a major bump is newer → none (major requires manual)', + releases: [rel('v2.0.0', daysAgo(10))], + current: '1.32.0', + cooloff: 72, + expect: null, + }, + { + name: 'same-major minor newer but inside cool-off → none', + releases: [rel('v1.33.0', hoursAgo(10))], + current: '1.32.0', + cooloff: 72, + expect: null, + }, + { + name: 'same-major patch past cool-off → selected', + releases: [rel('v1.32.1', daysAgo(5))], + current: '1.32.0', + cooloff: 72, + expect: '1.32.1', + }, + { + name: 'mixed: newest same-major past cool-off wins; major/in-cooloff/prerelease ignored', + releases: [ + rel('v2.0.0', daysAgo(30)), + rel('v1.34.0', hoursAgo(5)), + rel('v1.33.2', daysAgo(4)), + rel('v1.33.5', daysAgo(1), { prerelease: true }), + rel('v1.33.0', daysAgo(8)), + ], + current: '1.32.9', + cooloff: 72, + expect: '1.33.2', + }, + { + name: 'draft releases ignored', + releases: [rel('v1.33.0', daysAgo(5), { draft: true })], + current: '1.32.0', + cooloff: 72, + expect: null, + }, + { + name: 'malformed tag with injection chars → ignored (M2)', + releases: [rel('v1.33.0|; e reboot', daysAgo(10))], + current: '1.32.0', + cooloff: 72, + expect: null, + }, + { + name: 'dev build never updates', + releases: [rel('v1.33.0', daysAgo(10))], + current: 'dev', + cooloff: 72, + expect: null, + }, + { + name: 'cool-off of 0 applies immediately', + releases: [rel('v1.32.1', hoursAgo(1))], + current: '1.32.0', + cooloff: 0, + expect: '1.32.1', + }, + ] + + type WinCase = { name: string; start: string; end: string; at: string; expect: boolean } + const at = (hhmm: string) => `2026-06-04T${hhmm}:00` + const windows: WinCase[] = [ + { + name: 'normal 20:00-23:00 @ 21:00 → in', + start: '20:00', + end: '23:00', + at: at('21:00'), + expect: true, + }, + { + name: 'normal 20:00-23:00 @ 19:00 → out', + start: '20:00', + end: '23:00', + at: at('19:00'), + expect: false, + }, + { + name: 'wrap 22:00-02:00 @ 23:00 → in', + start: '22:00', + end: '02:00', + at: at('23:00'), + expect: true, + }, + { + name: 'wrap 22:00-02:00 @ 01:00 → in', + start: '22:00', + end: '02:00', + at: at('01:00'), + expect: true, + }, + { + name: 'wrap 22:00-02:00 @ 12:00 → out', + start: '22:00', + end: '02:00', + at: at('12:00'), + expect: false, + }, + ] + + let passed = 0 + let failed = 0 + + this.logger.log('') + this.logger.log('Eligibility scenarios:') + for (const c of eligibility) { + const got = svc.selectEligibleTarget(c.releases, c.current, c.cooloff, now) + const gotVersion = got ? got.version : null + const ok = gotVersion === c.expect + this.report(ok, `${c.name} (expected ${c.expect ?? 'none'}, got ${gotVersion ?? 'none'})`) + ok ? passed++ : failed++ + } + + this.logger.log('') + this.logger.log('Window scenarios:') + for (const c of windows) { + const cfg = { enabled: true, windowStart: c.start, windowEnd: c.end, cooloffHours: 72 } + const got = svc.isWithinWindow(cfg, DateTime.fromISO(c.at)) + const ok = got === c.expect + this.report(ok, `${c.name} (expected ${c.expect}, got ${got})`) + ok ? passed++ : failed++ + } + + this.logger.log('') + if (failed === 0) { + this.logger.success(`All ${passed} scenarios passed`) + } else { + this.logger.error(`${failed} scenario(s) failed, ${passed} passed`) + } + return failed === 0 + } + + private report(ok: boolean, message: string) { + if (ok) { + this.logger.log(` ${this.colors.green('✓')} ${message}`) + } else { + this.logger.log(` ${this.colors.red('✗')} ${message}`) + } + } +} diff --git a/admin/commands/queue/work.ts b/admin/commands/queue/work.ts index 49890e6..7e68122 100644 --- a/admin/commands/queue/work.ts +++ b/admin/commands/queue/work.ts @@ -9,6 +9,7 @@ import { RunBenchmarkJob } from '#jobs/run_benchmark_job' import { EmbedFileJob } from '#jobs/embed_file_job' import { CheckUpdateJob } from '#jobs/check_update_job' import { CheckServiceUpdatesJob } from '#jobs/check_service_updates_job' +import { AutoUpdateJob } from '#jobs/auto_update_job' export default class QueueWork extends BaseCommand { static commandName = 'queue:work' @@ -103,6 +104,7 @@ export default class QueueWork extends BaseCommand { // Schedule nightly update checks (idempotent, will persist over restarts) await CheckUpdateJob.scheduleNightly() await CheckServiceUpdatesJob.scheduleNightly() + await AutoUpdateJob.schedule() // Safety net: log unhandled rejections instead of crashing the worker process. // Individual job errors are already caught by BullMQ; this catches anything that @@ -133,6 +135,7 @@ export default class QueueWork extends BaseCommand { handlers.set(EmbedFileJob.key, new EmbedFileJob()) handlers.set(CheckUpdateJob.key, new CheckUpdateJob()) handlers.set(CheckServiceUpdatesJob.key, new CheckServiceUpdatesJob()) + handlers.set(AutoUpdateJob.key, new AutoUpdateJob()) queues.set(RunDownloadJob.key, RunDownloadJob.queue) queues.set(RunExtractPmtilesJob.key, RunExtractPmtilesJob.queue) @@ -141,6 +144,7 @@ export default class QueueWork extends BaseCommand { queues.set(EmbedFileJob.key, EmbedFileJob.queue) queues.set(CheckUpdateJob.key, CheckUpdateJob.queue) queues.set(CheckServiceUpdatesJob.key, CheckServiceUpdatesJob.queue) + queues.set(AutoUpdateJob.key, AutoUpdateJob.queue) return [handlers, queues] } diff --git a/admin/constants/kv_store.ts b/admin/constants/kv_store.ts index 6caeb72..da7a3cb 100644 --- a/admin/constants/kv_store.ts +++ b/admin/constants/kv_store.ts @@ -1,3 +1,3 @@ import { KVStoreKey } from "../types/kv_store.js"; -export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'rag.defaultIngestPolicy']; \ No newline at end of file +export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'rag.defaultIngestPolicy', 'autoUpdate.enabled', 'autoUpdate.windowStart', 'autoUpdate.windowEnd', 'autoUpdate.cooloffHours']; \ No newline at end of file diff --git a/admin/inertia/hooks/useAutoUpdateStatus.ts b/admin/inertia/hooks/useAutoUpdateStatus.ts new file mode 100644 index 0000000..57e5c22 --- /dev/null +++ b/admin/inertia/hooks/useAutoUpdateStatus.ts @@ -0,0 +1,11 @@ +import { useQuery } from '@tanstack/react-query' +import api from '~/lib/api' +import { AutoUpdateStatus } from '../../types/system' + +export const useAutoUpdateStatus = () => { + return useQuery({ + queryKey: ['auto-update-status'], + queryFn: () => api.getAutoUpdateStatus(), + refetchOnWindowFocus: false, + }) +} diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index d0db48e..10400a9 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -2,7 +2,7 @@ import axios, { AxiosError, AxiosInstance } from 'axios' import { ListRemoteZimFilesResponse, ListZimFilesResponse } from '../../types/zim' import { ServiceSlim } from '../../types/services' import { FileEntry } from '../../types/files' -import { CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus } from '../../types/system' +import { AutoUpdateStatus, CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus } from '../../types/system' import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads' import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps' import { EmbedJobWithProgress, FileWarningsResult, StoredFileInfo } from '../../types/rag' @@ -542,6 +542,13 @@ class API { })() } + async getAutoUpdateStatus() { + return catchInternal(async () => { + const response = await this.client.get('/system/auto-update/status') + return response.data + })() + } + async healthCheck() { return catchInternal(async () => { const response = await this.client.get<{ status: string }>('/health', { diff --git a/admin/inertia/pages/settings/update.tsx b/admin/inertia/pages/settings/update.tsx index 06db3bb..ffba3ba 100644 --- a/admin/inertia/pages/settings/update.tsx +++ b/admin/inertia/pages/settings/update.tsx @@ -15,6 +15,7 @@ import Switch from '~/components/inputs/Switch' import { useMutation, useQueryClient } from '@tanstack/react-query' import { useNotifications } from '~/context/NotificationContext' import { useSystemSetting } from '~/hooks/useSystemSetting' +import { useAutoUpdateStatus } from '~/hooks/useAutoUpdateStatus' import { formatBytes } from '~/lib/util' type Props = { @@ -257,6 +258,181 @@ function ContentUpdatesSection() { ) } +const COOLOFF_OPTIONS = [ + { value: 24, label: '24 hours (1 day)' }, + { value: 48, label: '48 hours (2 days)' }, + { value: 72, label: '72 hours (3 days)' }, + { value: 168, label: '7 days' }, +] + +function AutoUpdateSection() { + const { addNotification } = useNotifications() + const queryClient = useQueryClient() + const { data: status, isLoading } = useAutoUpdateStatus() + + const [windowStart, setWindowStart] = useState('02:00') + const [windowEnd, setWindowEnd] = useState('05:00') + const [cooloff, setCooloff] = useState(72) + + // Seed editable fields once the persisted status loads. + useEffect(() => { + if (status) { + setWindowStart(status.windowStart) + setWindowEnd(status.windowEnd) + setCooloff(status.cooloffHours) + } + }, [status?.windowStart, status?.windowEnd, status?.cooloffHours]) + + const saveMutation = useMutation({ + mutationFn: ({ key, value }: { key: string; value: any }) => api.updateSetting(key, value), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['auto-update-status'] }) + }, + onError: () => { + addNotification({ type: 'error', message: 'Failed to update auto-update setting.' }) + }, + }) + + const enabled = status?.enabled ?? false + const autoDisabled = !!status?.autoDisabledReason + + const handleToggle = (value: boolean) => { + saveMutation.mutate( + { key: 'autoUpdate.enabled', value }, + { + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['auto-update-status'] }) + addNotification({ + type: 'success', + message: value ? 'Automatic updates enabled.' : 'Automatic updates disabled.', + }) + }, + } + ) + } + + const handleSaveWindow = async () => { + try { + await api.updateSetting('autoUpdate.windowStart', windowStart) + await api.updateSetting('autoUpdate.windowEnd', windowEnd) + await api.updateSetting('autoUpdate.cooloffHours', String(cooloff)) + queryClient.invalidateQueries({ queryKey: ['auto-update-status'] }) + addNotification({ type: 'success', message: 'Auto-update schedule saved.' }) + } catch { + addNotification({ type: 'error', message: 'Failed to save auto-update schedule.' }) + } + } + + return ( + <> + +
+ {autoDisabled && ( + + )} + + + +
+ setWindowStart(e.target.value)} + disabled={!enabled} + helpText="Local server time" + /> + setWindowEnd(e.target.value)} + disabled={!enabled} + helpText="Local server time" + /> +
+ +

Delay after a release is published

+ +
+
+ +
+ + Save Schedule + +
+ + {enabled && status && ( +
+

+ Status: + {status.eligibleTarget + ? `Eligible update ready: ${status.eligibleTarget.version}` + : 'No eligible update — system is current or the latest release is a major version / still in cool-off.'} +

+

+ Update window: + {status.withinWindow ? 'Currently inside the window' : 'Currently outside the window'} +

+ {status.lastResult && ( +

+ Last check: + {status.lastResult} + {status.lastAttemptAt + ? ` (${new Date(status.lastAttemptAt).toLocaleString()})` + : ''} +

+ )} + {status.lastError && ( +

+ Last error: + {status.lastError} +

+ )} +
+ )} +
+ + ) +} + export default function SystemUpdatePage(props: { system: Props }) { const { addNotification } = useNotifications() @@ -668,6 +844,7 @@ export default function SystemUpdatePage(props: { system: Props }) { description="Receive release candidate (RC) versions before they are officially released. Note: RC versions may contain bugs and are not recommended for environments where stability and data integrity are critical." /> +
diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 598c951..0379b98 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -187,6 +187,7 @@ router router.post('/update', [SystemController, 'requestSystemUpdate']) router.get('/update/status', [SystemController, 'getSystemUpdateStatus']) router.get('/update/logs', [SystemController, 'getSystemUpdateLogs']) + router.get('/auto-update/status', [SystemController, 'getAutoUpdateStatus']) router.get('/settings', [SettingsController, 'getSetting']) router.patch('/settings', [SettingsController, 'updateSetting']) }) diff --git a/admin/types/kv_store.ts b/admin/types/kv_store.ts index e41f910..4193f57 100644 --- a/admin/types/kv_store.ts +++ b/admin/types/kv_store.ts @@ -7,6 +7,15 @@ export const KV_STORE_SCHEMA = { 'system.updateAvailable': 'boolean', 'system.latestVersion': 'string', 'system.earlyAccess': 'boolean', + 'autoUpdate.enabled': 'boolean', + 'autoUpdate.windowStart': 'string', + 'autoUpdate.windowEnd': 'string', + 'autoUpdate.cooloffHours': 'string', + 'autoUpdate.lastAttemptAt': 'string', + 'autoUpdate.lastError': 'string', + 'autoUpdate.lastResult': 'string', + 'autoUpdate.consecutiveFailures': 'string', + 'autoUpdate.autoDisabledReason': 'string', 'ui.hasVisitedEasySetup': 'boolean', 'ui.theme': 'string', 'ai.assistantCustomName': 'string', diff --git a/admin/types/system.ts b/admin/types/system.ts index c8ed4ab..265ea36 100644 --- a/admin/types/system.ts +++ b/admin/types/system.ts @@ -85,4 +85,25 @@ export type CheckLatestVersionResult = { currentVersion: string, latestVersion: string, message?: string +} + +export type AutoUpdateEligibleTarget = { + version: string + tag: string + publishedAt: string +} + +export type AutoUpdateStatus = { + enabled: boolean + windowStart: string + windowEnd: string + cooloffHours: number + currentVersion: string + withinWindow: boolean + eligibleTarget: AutoUpdateEligibleTarget | null + lastAttemptAt: string | null + lastResult: string | null + lastError: string | null + consecutiveFailures: number + autoDisabledReason: string | null } \ No newline at end of file