From 322087c1b79317bb04b1d1a97f4e0b65467b395d Mon Sep 17 00:00:00 2001 From: Ryanba <92616678+Gujiassh@users.noreply.github.com> Date: Wed, 29 Apr 2026 11:55:25 +0800 Subject: [PATCH 01/18] fix(UI): improve global map banner display logic (#702) --- admin/inertia/lib/global_map_banner.ts | 10 ++++++ admin/inertia/pages/settings/maps.tsx | 20 +++++++++++- admin/tests/unit/global_map_banner.spec.ts | 37 ++++++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 admin/inertia/lib/global_map_banner.ts create mode 100644 admin/tests/unit/global_map_banner.spec.ts diff --git a/admin/inertia/lib/global_map_banner.ts b/admin/inertia/lib/global_map_banner.ts new file mode 100644 index 0000000..1693dac --- /dev/null +++ b/admin/inertia/lib/global_map_banner.ts @@ -0,0 +1,10 @@ +export function hasDownloadedGlobalMap( + globalMapKey: string | null | undefined, + storedMapFiles: Array<{ name: string }> +): boolean { + if (!globalMapKey) { + return false + } + + return storedMapFiles.some((file) => file.name === globalMapKey || /^\d{8}\.pmtiles$/.test(file.name)) +} diff --git a/admin/inertia/pages/settings/maps.tsx b/admin/inertia/pages/settings/maps.tsx index 0212931..da97bb1 100644 --- a/admin/inertia/pages/settings/maps.tsx +++ b/admin/inertia/pages/settings/maps.tsx @@ -17,6 +17,7 @@ import type { CollectionWithStatus } from '../../../types/collections' import ActiveDownloads from '~/components/ActiveDownloads' import Alert from '~/components/Alert' import { formatBytes } from '~/lib/util' +import { hasDownloadedGlobalMap } from '~/lib/global_map_banner' const CURATED_COLLECTIONS_KEY = 'curated-map-collections' const GLOBAL_MAP_INFO_KEY = 'global-map-info' @@ -45,6 +46,7 @@ export default function MapsManager(props: { queryFn: () => api.getGlobalMapInfo(), refetchOnWindowFocus: false, }) + const globalMapAlreadyDownloaded = hasDownloadedGlobalMap(globalMapInfo?.key, props.maps.regionFiles) const downloadGlobalMap = useMutation({ mutationFn: () => api.downloadGlobalMap(), @@ -251,7 +253,23 @@ export default function MapsManager(props: { }} /> )} - {globalMapInfo && ( + {globalMapInfo && globalMapAlreadyDownloaded && ( + confirmGlobalMapDownload(), + }} + /> + )} + {globalMapInfo && !globalMapAlreadyDownloaded && ( { + assert.equal( + hasDownloadedGlobalMap('20260402.pmtiles', [ + { name: '20260402.pmtiles' }, + { name: 'california.pmtiles' }, + ]), + true + ) +}) + +test('returns false when the global map key is missing', () => { + assert.equal( + hasDownloadedGlobalMap('20260402.pmtiles', [ + { name: 'california.pmtiles' }, + ]), + false + ) +}) + +test('returns true when an older global map build is already on disk', () => { + assert.equal( + hasDownloadedGlobalMap('20260402.pmtiles', [ + { name: '20260315.pmtiles' }, + { name: 'california.pmtiles' }, + ]), + true + ) +}) + +test('returns false when there is no global map info', () => { + assert.equal(hasDownloadedGlobalMap(undefined, [{ name: '20260402.pmtiles' }]), false) +}) From 5924056502f098a6c6271edd6970689e04a44377 Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Tue, 28 Apr 2026 21:53:56 -0700 Subject: [PATCH 02/18] feat(AI): improved AMD GPU acceleration for Ollama via ROCm + HSA override (#804) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(AI): re-enable AMD GPU acceleration for Ollama via ROCm + HSA override Re-enables AMD GPU support that was disabled in 77f1868 pending validation of the ROCm image and device discovery. Validation done 2026-04-28 on a Minisforum UM890 Pro (Ryzen 9 PRO 8945HS + Radeon 780M iGPU) — Ollama correctly offloaded all model layers to the iGPU when the container was started with /dev/kfd + /dev/dri passthrough and HSA_OVERRIDE_GFX_VERSION=11.0.0. On llama3.2:1b, GPU inference ran at 51.83 tok/s vs 33.16 tok/s on CPU (same hardware, same prompt) — a 1.56x speedup confirmed by Ollama logs showing "load_tensors: offloaded 17/17 layers to GPU". Changes ------- docker_service.ts - Restore _discoverAMDDevices() (simplified — pass /dev/dri as a directory entry, mirroring `docker run --device /dev/dri` behavior, instead of the prior brittle hardcoded card0/renderD128 fallback that broke on systems where the AMD GPU enumerates as card1+). - Restore the AMD branch in _createContainer(): - Switches Ollama image to ollama/ollama:rocm - Mounts /dev/kfd + /dev/dri via Devices - Sets HSA_OVERRIDE_GFX_VERSION=11.0.0 (required for unsupported-but-RDNA3 iGPUs like gfx1103; harmless on supported discrete cards) - KV opt-out via ai.amdGpuAcceleration (default on) - Mirror the AMD branch in updateContainer(): - Lifted GPU detection above docker.pull() so AMD updates pull :rocm rather than the standard :targetVersion tag (per-version ROCm tags aren't always published) - Replaces stale HSA_OVERRIDE in the inspect-captured env on update, so containers built before this PR pick up the current value system_service.ts - New getOllamaInferenceComputeFromLogs() — parses Ollama startup log line "msg=\"inference compute\" ... library=CUDA|ROCm ..." which Ollama emits for both NVIDIA and AMD. Catches silent CPU fallback (e.g. NVML death after update, or HSA_OVERRIDE failure) that the prior nvidia-smi exec probe couldn't detect. - gpuHealth refactored to use log parsing as the primary probe for both vendors, with nvidia-smi exec retained as the NVIDIA-only secondary path for hardware enrichment when log parsing has no startup line yet. - AMD path uses gpu.type KV value (persisted by DockerService._detectGPUType) + ai.amdGpuAcceleration opt-out to determine hasRocmRuntime. types/system.ts - GpuHealthStatus extended additively: hasRocmRuntime + optional gpuVendor. types/kv_store.ts - New ai.amdGpuAcceleration boolean (default-on). settings/models.tsx, settings/system.tsx - passthrough_failed banner copy now reads vendor from gpuHealth.gpuVendor ("an AMD GPU" vs "an NVIDIA GPU"). Same Fix button hits the same force-reinstall endpoint, which now configures AMD correctly. install_nomad.sh - AMD detection in verify_gpu_setup() upgraded from a strict-positive "ROCm not currently available" message to "ROCm acceleration will be configured automatically." Also tightens the lspci match to display controller classes (avoids false positives from AMD CPU host bridges, matching the same fix already in DockerService._detectGPUType). Auto-remediation ---------------- Issue #755 proposes auto-remediation when gpuHealth.status flips to passthrough_failed (today the user has to click "Fix: Reinstall AI Assistant"). When that PR lands, AMD coverage falls out for free since this PR uses the same passthrough_failed status code via the shared gpuHealth machinery — #755's guard will need to flip from hasNvidiaRuntime === true to (hasNvidiaRuntime || hasRocmRuntime). Closes #124 (AMD GPU support). Co-Authored-By: Claude Opus 4.7 (1M context) * fix(AI): detect AMD GPU presence inside admin container via marker file The admin container doesn't have lspci installed, and AMD GPUs don't register a Docker runtime the way NVIDIA does — so DockerService._detectGPUType() and SystemService.gpuHealth had no way to know an AMD GPU was present. The previous implementation fell through to lspci, which silently failed inside the admin container, leaving gpu.type unset and gpuHealth stuck at 'no_gpu' even on systems with an AMD GPU. (NVIDIA worked because Docker registers the nvidia runtime, which is reachable via dockerInfo.Runtimes from any container.) Discovered while testing the AMD acceleration patch on a Minisforum UM890 Pro: the AMD branch in _createContainer() never fired because _detectGPUType() returned 'none' even on a host with a working /dev/kfd. Fix --- install_nomad.sh writes the host-detected GPU type ('nvidia' | 'amd') to a marker file in the storage volume the admin container already bind-mounts: /opt/project-nomad/storage/.nomad-gpu-type → /app/storage/.nomad-gpu-type DockerService._detectGPUType() reads the marker as a secondary probe (after the Docker runtime check) — covers AMD detection from inside the container without requiring lspci or a /dev bind mount. SystemService falls back to the marker file when KV gpu.type is empty so the System page reflects AMD presence even before the user installs AI Assistant for the first time. (Without this, the page would say 'no_gpu' until Ollama was installed, even on hosts with an AMD GPU detected at install time.) Verified on NOMAD6 (UM890 Pro, Ubuntu 24.04, 780M iGPU): with the marker file in place and admin restarted, the patch's AMD branch fires correctly on Force Reinstall AI Assistant. Resulting nomad_ollama runs ollama/ollama:rocm with /dev/kfd + /dev/dri passthrough and HSA_OVERRIDE_GFX_VERSION=11.0.0; Ollama logs show 'library=ROCm compute=gfx1100 ... type=iGPU'. NOMAD's in-product benchmark on the same hardware climbed from 33.8 tok/s (CPU) to 57.3 tok/s (GPU) — a 1.69x speedup, with TTFT dropping from 148ms to 66ms. Migration for existing AMD installs ----------------------------------- Users on an existing NOMAD install with an AMD GPU have no marker file (the install script wrote it on a fresh install). Two paths get them on the GPU: 1. Re-run install_nomad.sh — writes the marker, no other side effects 2. Manually: echo amd | sudo tee /opt/project-nomad/storage/.nomad-gpu-type Either then triggers AMD detection on the next AI Assistant install/reinstall. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(AI): pull ollama/ollama:rocm separately when AMD branch overrides image The pull-if-missing logic in _createContainer ran against service.container_image (the DB-pinned tag, e.g. ollama/ollama:0.18.2). The AMD branch then overrode finalImage to ollama/ollama:rocm — but if that image wasn't already local, the container creation step failed with "no such image: ollama/ollama:rocm". Caught while validating on NOMAD2 (Ryzen AI 9 HX 370 + Radeon 890M / RDNA 3.5): the prior end-to-end test on NOMAD6 had silently passed because the rocm image was already pulled there from an earlier sidecar test, masking the bug. Fix: inside the AMD branch, after setting finalImage to ollama/ollama:rocm, run a parallel _checkImageExists + docker.pull dance for the new tag. Also confirmed via this validation: the same HSA_OVERRIDE_GFX_VERSION=11.0.0 override works on the 890M (gfx1150 / RDNA 3.5) — Ollama logs report 'library=ROCm compute=gfx1100 description="AMD Radeon 890M Graphics"' and inference runs at 51.68 tok/s (matching the existing X1 Pro published tile of 51.7 tok/s on the same hardware class). RDNA 3 (780M, gfx1103) and RDNA 3.5 (890M, gfx1150) both use the same override successfully. Co-Authored-By: Claude Opus 4.7 (1M context) * build(Dockerfile): include pciutils for lspci gpu detection fallback --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Jake Turner --- Dockerfile | 9 +- admin/app/services/docker_service.ts | 257 ++++++++++++++---------- admin/app/services/system_service.ts | 187 ++++++++++++++--- admin/inertia/pages/settings/models.tsx | 2 +- admin/inertia/pages/settings/system.tsx | 2 +- admin/types/kv_store.ts | 1 + admin/types/system.ts | 2 + install/install_nomad.sh | 27 ++- 8 files changed, 343 insertions(+), 144 deletions(-) diff --git a/Dockerfile b/Dockerfile index 03acaa9..8850e23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,14 @@ FROM node:22-slim AS base # Install bash & curl for entrypoint script compatibility, graphicsmagick for pdf2pic, and vips-dev & build-base for sharp -RUN apt-get update && apt-get install -y bash curl graphicsmagick libvips-dev build-essential +RUN apt-get update && apt-get install -y \ + bash \ + curl \ + graphicsmagick \ + libvips-dev \ + build-essential \ + pciutils \ + && rm -rf /var/lib/apt/lists/* # All deps stage FROM base AS deps diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts index 714cd11..7501e78 100644 --- a/admin/app/services/docker_service.ts +++ b/admin/app/services/docker_service.ts @@ -10,7 +10,7 @@ import { KiwixLibraryService } from './kiwix_library_service.js' import { SERVICE_NAMES } from '../../constants/service_names.js' import { exec } from 'child_process' import { promisify } from 'util' -// import { readdir } from 'fs/promises' +import { readFile } from 'node:fs/promises' import KVStore from '#models/kv_store' import { BROADCAST_CHANNELS } from '../../constants/broadcast.js' import { KIWIX_LIBRARY_CMD } from '../../constants/kiwix.js' @@ -500,6 +500,7 @@ export class DockerService { // GPU-aware configuration for Ollama let finalImage = service.container_image let gpuHostConfig = containerConfig?.HostConfig || {} + let amdGpuConfigured = false if (service.service_name === SERVICE_NAMES.OLLAMA) { const gpuResult = await this._detectGPUType() @@ -523,16 +524,51 @@ export class DockerService { ], } } else if (gpuResult.type === 'amd') { - this._broadcast( - service.service_name, - 'gpu-config', - `AMD GPU detected. ROCm GPU acceleration is not yet supported in this version — proceeding with CPU-only configuration. GPU support for AMD will be available in a future update.` - ) - logger.warn('[DockerService] AMD GPU detected but ROCm support is not yet enabled. Using CPU-only configuration.') - // TODO: Re-enable AMD GPU support once ROCm image and device discovery are validated. - // When re-enabling: - // 1. Switch image to 'ollama/ollama:rocm' - // 2. Restore _discoverAMDDevices() to map /dev/kfd and /dev/dri/* into the container + // AMD acceleration is opt-out via the 'ai.amdGpuAcceleration' KV key (default-on). + // Per memory feedback: KV values can be string or boolean — coerce explicitly. + const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration') + const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false' + + if (amdAccelerationEnabled) { + this._broadcast( + service.service_name, + 'gpu-config', + `AMD GPU detected. Using ROCm image with /dev/kfd and /dev/dri passthrough...` + ) + + finalImage = 'ollama/ollama:rocm' + + // The pull-if-missing earlier in this function used service.container_image + // (the DB-pinned tag, e.g. ollama/ollama:0.18.2). The AMD branch overrides + // to a different tag — so we need to pull :rocm separately if it's not local. + const rocmImageExists = await this._checkImageExists(finalImage) + if (!rocmImageExists) { + this._broadcast( + service.service_name, + 'pulling', + `Pulling Docker image ${finalImage}...` + ) + const rocmPullStream = await this.docker.pull(finalImage) + await new Promise((res) => this.docker.modem.followProgress(rocmPullStream, res)) + } + + const amdDevices = await this._discoverAMDDevices() + gpuHostConfig = { + ...gpuHostConfig, + Devices: amdDevices, + } + amdGpuConfigured = true + logger.info( + `[DockerService] Configured ROCm image and ${amdDevices.length} AMD device entries for Ollama` + ) + } else { + this._broadcast( + service.service_name, + 'gpu-config', + `AMD GPU detected but acceleration is disabled via ai.amdGpuAcceleration. Using CPU-only configuration.` + ) + logger.info('[DockerService] AMD GPU acceleration disabled by KV opt-out; using CPU-only configuration.') + } } else if (gpuResult.toolkitMissing) { this._broadcast( service.service_name, @@ -555,6 +591,12 @@ export class DockerService { if (flashAttentionEnabled !== false) { ollamaEnv.push('OLLAMA_FLASH_ATTENTION=1') } + if (amdGpuConfigured) { + // RDNA3 iGPUs (gfx1103: 780M, 880M, 890M, ...) aren't on AMD's official ROCm + // allowlist but work when forced to identify as gfx1100 via HSA_OVERRIDE_GFX_VERSION. + // Harmless on supported discrete cards (gfx1030 RX 6800, etc.) — they ignore the override. + ollamaEnv.push('HSA_OVERRIDE_GFX_VERSION=11.0.0') + } } this._broadcast( @@ -857,7 +899,10 @@ export class DockerService { /** * Detect GPU type and toolkit availability. * Primary: Check Docker runtimes via docker.info() (works from inside containers). - * Fallback: lspci for host-based installs and AMD detection. + * Secondary: Read /app/storage/.nomad-gpu-type written by install_nomad.sh — needed + * for AMD detection because lspci isn't available inside the admin container and + * AMD has no Docker runtime registration to query. + * Fallback: lspci for host-based installs. */ private async _detectGPUType(): Promise<{ type: 'nvidia' | 'amd' | 'none'; toolkitMissing?: boolean }> { try { @@ -874,6 +919,24 @@ export class DockerService { logger.warn(`[DockerService] Could not query Docker info for GPU runtimes: ${error.message}`) } + // Secondary: install_nomad.sh writes the host-detected GPU type to a marker file in + // the storage volume so the admin container (which lacks lspci) can read it. + try { + const marker = (await readFile('/app/storage/.nomad-gpu-type', 'utf8')).trim() + if (marker === 'nvidia') { + // Hardware present but Docker doesn't have nvidia runtime → toolkit missing + logger.warn('[DockerService] NVIDIA GPU recorded in marker file but NVIDIA Container Toolkit is not installed') + return { type: 'none', toolkitMissing: true } + } + if (marker === 'amd') { + logger.info('[DockerService] AMD GPU detected via install-time marker file') + await this._persistGPUType('amd') + return { type: 'amd' } + } + } catch { + // No marker file — fall through to lspci attempt for host-based installs + } + // Fallback: lspci for host-based installs (not available inside Docker) const execAsync = promisify(exec) @@ -937,60 +1000,23 @@ export class DockerService { } /** - * Discover AMD GPU DRI devices dynamically. - * Returns an array of device configurations for Docker. + * Build the Docker Devices array for AMD GPU passthrough. + * + * Returns /dev/kfd (Kernel Fusion Driver, required by ROCm) and /dev/dri (the DRM + * device tree). Passing /dev/dri as a single directory entry mirrors Docker CLI + * --device behavior — the daemon expands it to all child devices (card*, renderD*) + * regardless of how the host enumerates them. This avoids the brittle hardcoded + * fallback (card0/renderD128) the prior implementation used, which was wrong on + * systems where the AMD GPU enumerates as card1+ (e.g. UM890 Pro 780M iGPU). */ - // private async _discoverAMDDevices(): Promise< - // Array<{ PathOnHost: string; PathInContainer: string; CgroupPermissions: string }> - // > { - // try { - // const devices: Array<{ - // PathOnHost: string - // PathInContainer: string - // CgroupPermissions: string - // }> = [] - - // // Always add /dev/kfd (Kernel Fusion Driver) - // devices.push({ - // PathOnHost: '/dev/kfd', - // PathInContainer: '/dev/kfd', - // CgroupPermissions: 'rwm', - // }) - - // // Discover DRI devices in /dev/dri/ - // try { - // const driDevices = await readdir('/dev/dri') - // for (const device of driDevices) { - // const devicePath = `/dev/dri/${device}` - // devices.push({ - // PathOnHost: devicePath, - // PathInContainer: devicePath, - // CgroupPermissions: 'rwm', - // }) - // } - // logger.info( - // `[DockerService] Discovered ${driDevices.length} DRI devices: ${driDevices.join(', ')}` - // ) - // } catch (error) { - // logger.warn(`[DockerService] Could not read /dev/dri directory: ${error.message}`) - // // Fallback to common device names if directory read fails - // const fallbackDevices = ['card0', 'renderD128'] - // for (const device of fallbackDevices) { - // devices.push({ - // PathOnHost: `/dev/dri/${device}`, - // PathInContainer: `/dev/dri/${device}`, - // CgroupPermissions: 'rwm', - // }) - // } - // logger.info(`[DockerService] Using fallback DRI devices: ${fallbackDevices.join(', ')}`) - // } - - // return devices - // } catch (error) { - // logger.error(`[DockerService] Error discovering AMD devices: ${error.message}`) - // return [] - // } - // } + private async _discoverAMDDevices(): Promise< + Array<{ PathOnHost: string; PathInContainer: string; CgroupPermissions: string }> + > { + return [ + { PathOnHost: '/dev/kfd', PathInContainer: '/dev/kfd', CgroupPermissions: 'rwm' }, + { PathOnHost: '/dev/dri', PathInContainer: '/dev/dri', CgroupPermissions: 'rwm' }, + ] + } /** * Update a service container to a new image version while preserving volumes and data. @@ -1014,12 +1040,60 @@ export class DockerService { this.activeInstallations.add(serviceName) - // Compute new image string + // Compute new image string. AMD-on-Ollama overrides this to the rolling :rocm tag + // (set during GPU detection below) since per-version ROCm tags aren't always published. const currentImage = service.container_image const imageBase = currentImage.includes(':') ? currentImage.substring(0, currentImage.lastIndexOf(':')) : currentImage - const newImage = `${imageBase}:${targetVersion}` + let newImage = `${imageBase}:${targetVersion}` + + // GPU detection runs before the pull so AMD updates pull ollama/ollama:rocm rather + // than the standard tag. Detection result is reused below when building the new + // container config (devices, env). Non-Ollama services skip this entirely. + let updatedDeviceRequests: any[] | undefined = undefined + let updatedAmdDevices: any[] | undefined = undefined + let updatedAmdGpuConfigured = false + if (serviceName === SERVICE_NAMES.OLLAMA) { + const gpuResult = await this._detectGPUType() + if (gpuResult.type === 'nvidia') { + this._broadcast( + serviceName, + 'update-gpu-config', + `NVIDIA container runtime detected. Configuring updated container with GPU support...` + ) + updatedDeviceRequests = [ + { Driver: 'nvidia', Count: -1, Capabilities: [['gpu']] }, + ] + } else if (gpuResult.type === 'amd') { + const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration') + const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false' + if (amdAccelerationEnabled) { + this._broadcast( + serviceName, + 'update-gpu-config', + `AMD GPU detected. Using ROCm image with /dev/kfd and /dev/dri passthrough...` + ) + newImage = 'ollama/ollama:rocm' + updatedAmdDevices = await this._discoverAMDDevices() + updatedAmdGpuConfigured = true + } else { + this._broadcast( + serviceName, + 'update-gpu-config', + `AMD GPU detected but acceleration is disabled via ai.amdGpuAcceleration. Using CPU-only configuration.` + ) + } + } else if (gpuResult.toolkitMissing) { + this._broadcast( + serviceName, + 'update-gpu-config', + `NVIDIA GPU detected but NVIDIA Container Toolkit is not installed. Using CPU-only configuration. Install the toolkit and reinstall AI Assistant for GPU acceleration: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html` + ) + } else { + this._broadcast(serviceName, 'update-gpu-config', `No GPU detected. Using CPU-only configuration.`) + } + } // Step 1: Pull new image this._broadcast(serviceName, 'update-pulling', `Pulling image ${newImage}...`) @@ -1054,48 +1128,21 @@ export class DockerService { const hostConfig = inspectData.HostConfig || {} - // Re-run GPU detection for Ollama so updates always reflect the current GPU environment. - // This handles cases where the NVIDIA Container Toolkit was installed after the initial - // Ollama setup, and ensures DeviceRequests are always built fresh rather than relying on - // round-tripping the Docker inspect format back into the create API. - let updatedDeviceRequests: any[] | undefined = undefined - if (serviceName === SERVICE_NAMES.OLLAMA) { - const gpuResult = await this._detectGPUType() - - if (gpuResult.type === 'nvidia') { - this._broadcast( - serviceName, - 'update-gpu-config', - `NVIDIA container runtime detected. Configuring updated container with GPU support...` - ) - updatedDeviceRequests = [ - { - Driver: 'nvidia', - Count: -1, - Capabilities: [['gpu']], - }, + // GPU detection already ran above (before the pull) so we know the right image, devices, + // and whether HSA_OVERRIDE needs injection. For AMD, replace any prior HSA_OVERRIDE in + // the inspect-captured env so updates from older containers pick up the current value. + const baseEnv = inspectData.Config?.Env || [] + const finalEnv = updatedAmdGpuConfigured + ? [ + ...baseEnv.filter((e: string) => !e.startsWith('HSA_OVERRIDE_GFX_VERSION=')), + 'HSA_OVERRIDE_GFX_VERSION=11.0.0', ] - } else if (gpuResult.type === 'amd') { - this._broadcast( - serviceName, - 'update-gpu-config', - `AMD GPU detected. ROCm GPU acceleration is not yet supported — using CPU-only configuration.` - ) - } else if (gpuResult.toolkitMissing) { - this._broadcast( - serviceName, - 'update-gpu-config', - `NVIDIA GPU detected but NVIDIA Container Toolkit is not installed. Using CPU-only configuration. Install the toolkit and reinstall AI Assistant for GPU acceleration: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html` - ) - } else { - this._broadcast(serviceName, 'update-gpu-config', `No GPU detected. Using CPU-only configuration.`) - } - } + : baseEnv const newContainerConfig: any = { Image: newImage, name: serviceName, - Env: inspectData.Config?.Env || undefined, + Env: finalEnv.length > 0 ? finalEnv : undefined, Cmd: inspectData.Config?.Cmd || undefined, ExposedPorts: inspectData.Config?.ExposedPorts || undefined, WorkingDir: inspectData.Config?.WorkingDir || undefined, @@ -1105,7 +1152,7 @@ export class DockerService { PortBindings: hostConfig.PortBindings || undefined, RestartPolicy: hostConfig.RestartPolicy || undefined, DeviceRequests: serviceName === SERVICE_NAMES.OLLAMA ? updatedDeviceRequests : (hostConfig.DeviceRequests || undefined), - Devices: hostConfig.Devices || undefined, + Devices: serviceName === SERVICE_NAMES.OLLAMA && updatedAmdDevices ? updatedAmdDevices : (hostConfig.Devices || undefined), }, NetworkingConfig: inspectData.NetworkSettings?.Networks ? { diff --git a/admin/app/services/system_service.ts b/admin/app/services/system_service.ts index 5701de3..1a55cfb 100644 --- a/admin/app/services/system_service.ts +++ b/admin/app/services/system_service.ts @@ -12,6 +12,7 @@ import { } from '../../types/system.js' import { SERVICE_NAMES } from '../../constants/service_names.js' import { readFileSync } from 'node:fs' +import { readFile } from 'node:fs/promises' import path, { join } from 'node:path' import { getAllFilesystems, getFile } from '../utils/fs.js' import axios from 'axios' @@ -72,6 +73,61 @@ export class SystemService { return false } + /** + * Probe Ollama startup logs for the canonical "inference compute" line that records + * which compute backend was selected. This catches silent CPU fallback (e.g. when + * /dev/kfd is mounted but ROCm initialization fails, or NVML dies after an update) + * which the older nvidia-smi exec probe could not detect. + * + * Returns the parsed library, GPU model name, and VRAM in MiB, or null when: + * - the Ollama container is not running + * - the line has not been emitted (Ollama still starting up) + * - logs show CPU-only operation (no GPU detected) + */ + async getOllamaInferenceComputeFromLogs(): Promise<{ + library: 'CUDA' | 'ROCm' + name: string + vramMiB: number + } | null> { + try { + const containers = await this.dockerService.docker.listContainers({ all: false }) + const ollamaContainer = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`)) + if (!ollamaContainer) return null + + const container = this.dockerService.docker.getContainer(ollamaContainer.Id) + const buf = (await container.logs({ + stdout: true, + stderr: true, + tail: 500, + follow: false, + })) as unknown as Buffer + const logs = buf.toString('utf8') + + const lines = logs.split('\n').filter((l) => l.includes('msg="inference compute"')) + if (lines.length === 0) return null + + const lastLine = lines[lines.length - 1] + const libraryMatch = lastLine.match(/library=(CUDA|ROCm)/) + if (!libraryMatch) return null + + const descMatch = lastLine.match(/description="([^"]+)"/) + const totalMatch = lastLine.match(/total="([0-9.]+)\s*GiB"/) + + return { + library: libraryMatch[1] as 'CUDA' | 'ROCm', + name: + descMatch?.[1] || + (libraryMatch[1] === 'CUDA' ? 'NVIDIA GPU' : 'AMD GPU'), + vramMiB: totalMatch ? Math.round(Number.parseFloat(totalMatch[1]) * 1024) : 0, + } + } catch (error) { + logger.warn( + `[SystemService] Failed to probe Ollama logs for inference compute line: ${error instanceof Error ? error.message : error}` + ) + return null + } + } + async getNvidiaSmiInfo(): Promise< | Array<{ vendor: string; model: string; vram: number }> | { error: string } @@ -317,10 +373,14 @@ export class SystemService { logger.error('Error reading disk info file:', error) } - // GPU health tracking — detect when host has NVIDIA GPU but Ollama can't access it + // GPU health tracking — detect when host has a GPU runtime but Ollama can't access it. + // Primary probe: parse Ollama's "inference compute" startup log line for both NVIDIA + // and AMD. Secondary probe (NVIDIA only): nvidia-smi exec, retained as a fallback for + // hardware enrichment when log parsing has not yet captured a startup line. let gpuHealth: GpuHealthStatus = { status: 'no_gpu', hasNvidiaRuntime: false, + hasRocmRuntime: false, ollamaGpuAccessible: false, } @@ -340,27 +400,51 @@ export class SystemService { } // If si.graphics() returned no controllers (common inside Docker), - // fall back to nvidia runtime + nvidia-smi detection + // fall back to runtime + Ollama log probe to figure out what's accessible. if (!graphics.controllers || graphics.controllers.length === 0) { const runtimes = dockerInfo.Runtimes || {} - if ('nvidia' in runtimes) { - gpuHealth.hasNvidiaRuntime = true - const nvidiaInfo = await this.getNvidiaSmiInfo() - if (Array.isArray(nvidiaInfo)) { - graphics.controllers = nvidiaInfo.map((gpu) => ({ - model: gpu.model, - vendor: gpu.vendor, - bus: '', - vram: gpu.vram, - vramDynamic: false, // assume false here, we don't actually use this field for our purposes. - })) + gpuHealth.hasNvidiaRuntime = 'nvidia' in runtimes + + // AMD doesn't register a Docker runtime. Detection sources, in priority order: + // 1. KV 'gpu.type' (set by DockerService._detectGPUType after first Ollama install) + // 2. Marker file at /app/storage/.nomad-gpu-type (written by install_nomad.sh) + // The marker file matters because the System page should reflect AMD presence + // even before AI Assistant has been installed for the first time. + let savedGpuType: string | null | undefined = await KVStore.getValue('gpu.type') as string | undefined + if (!savedGpuType) { + try { + savedGpuType = (await readFile('/app/storage/.nomad-gpu-type', 'utf8')).trim() + } catch {} + } + const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration') + const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false' + gpuHealth.hasRocmRuntime = savedGpuType === 'amd' && amdAccelerationEnabled + + if (gpuHealth.hasNvidiaRuntime || gpuHealth.hasRocmRuntime) { + gpuHealth.gpuVendor = gpuHealth.hasNvidiaRuntime ? 'nvidia' : 'amd' + + // Primary probe: Ollama log parsing — works for both vendors and catches silent fallback + const logInfo = await this.getOllamaInferenceComputeFromLogs() + if (logInfo) { + graphics.controllers = [ + { + model: logInfo.name, + vendor: logInfo.library === 'CUDA' ? 'NVIDIA' : 'AMD', + bus: '', + vram: logInfo.vramMiB, + vramDynamic: false, + }, + ] gpuHealth.status = 'ok' gpuHealth.ollamaGpuAccessible = true - } else if (nvidiaInfo === 'OLLAMA_NOT_FOUND') { - // No local Ollama container — check if a remote Ollama URL is configured - const externalOllamaGpu = await this.getExternalOllamaGpuInfo() - if (externalOllamaGpu) { - graphics.controllers = externalOllamaGpu.map((gpu) => ({ + } else if (gpuHealth.hasNvidiaRuntime) { + // NVIDIA secondary path: nvidia-smi exec preserves prior behavior when + // the log parser hasn't seen a startup line yet (e.g. log rotation, + // very fresh container). Distinguishes "no Ollama container" from + // "container exists but GPU broken". + const nvidiaInfo = await this.getNvidiaSmiInfo() + if (Array.isArray(nvidiaInfo)) { + graphics.controllers = nvidiaInfo.map((gpu) => ({ model: gpu.model, vendor: gpu.vendor, bus: '', @@ -369,25 +453,66 @@ export class SystemService { })) gpuHealth.status = 'ok' gpuHealth.ollamaGpuAccessible = true + } else if (nvidiaInfo === 'OLLAMA_NOT_FOUND') { + const externalOllamaGpu = await this.getExternalOllamaGpuInfo() + if (externalOllamaGpu) { + graphics.controllers = externalOllamaGpu.map((gpu) => ({ + model: gpu.model, + vendor: gpu.vendor, + bus: '', + vram: gpu.vram, + vramDynamic: false, + })) + gpuHealth.status = 'ok' + gpuHealth.ollamaGpuAccessible = true + } else { + gpuHealth.status = 'ollama_not_installed' + } } else { - gpuHealth.status = 'ollama_not_installed' + const externalOllamaGpu = await this.getExternalOllamaGpuInfo() + if (externalOllamaGpu) { + graphics.controllers = externalOllamaGpu.map((gpu) => ({ + model: gpu.model, + vendor: gpu.vendor, + bus: '', + vram: gpu.vram, + vramDynamic: false, + })) + gpuHealth.status = 'ok' + gpuHealth.ollamaGpuAccessible = true + } else { + gpuHealth.status = 'passthrough_failed' + logger.warn( + `NVIDIA runtime detected but GPU passthrough failed: ${typeof nvidiaInfo === 'string' ? nvidiaInfo : JSON.stringify(nvidiaInfo)}` + ) + } } } else { - const externalOllamaGpu = await this.getExternalOllamaGpuInfo() - if (externalOllamaGpu) { - graphics.controllers = externalOllamaGpu.map((gpu) => ({ - model: gpu.model, - vendor: gpu.vendor, - bus: '', - vram: gpu.vram, - vramDynamic: false, - })) - gpuHealth.status = 'ok' - gpuHealth.ollamaGpuAccessible = true + // AMD path: no nvidia-smi equivalent worth running — log parser is authoritative. + // Distinguish "Ollama not running" from "Ollama running but no GPU log line". + const containers = await this.dockerService.docker.listContainers({ all: false }) + const ollamaRunning = containers.some((c) => + c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`) + ) + if (!ollamaRunning) { + const externalOllamaGpu = await this.getExternalOllamaGpuInfo() + if (externalOllamaGpu) { + graphics.controllers = externalOllamaGpu.map((gpu) => ({ + model: gpu.model, + vendor: gpu.vendor, + bus: '', + vram: gpu.vram, + vramDynamic: false, + })) + gpuHealth.status = 'ok' + gpuHealth.ollamaGpuAccessible = true + } else { + gpuHealth.status = 'ollama_not_installed' + } } else { gpuHealth.status = 'passthrough_failed' logger.warn( - `NVIDIA runtime detected but GPU passthrough failed: ${typeof nvidiaInfo === 'string' ? nvidiaInfo : JSON.stringify(nvidiaInfo)}` + 'AMD GPU detected but Ollama logs show no ROCm initialization — passthrough or HSA override may have failed' ) } } diff --git a/admin/inertia/pages/settings/models.tsx b/admin/inertia/pages/settings/models.tsx index fc2b1dc..fe119d8 100644 --- a/admin/inertia/pages/settings/models.tsx +++ b/admin/inertia/pages/settings/models.tsx @@ -283,7 +283,7 @@ export default function ModelsPage(props: { type="warning" variant="bordered" title="GPU Not Accessible" - message={`Your system has an NVIDIA GPU, but ${aiAssistantName} can't access it. AI is running on CPU only, which is significantly slower.`} + message={`Your system has ${systemInfo?.gpuHealth?.gpuVendor === 'amd' ? 'an AMD' : 'an NVIDIA'} GPU, but ${aiAssistantName} can't access it. AI is running on CPU only, which is significantly slower.`} className="!mt-6" dismissible={true} onDismiss={handleDismissGpuBanner} diff --git a/admin/inertia/pages/settings/system.tsx b/admin/inertia/pages/settings/system.tsx index 7b40088..1a13b52 100644 --- a/admin/inertia/pages/settings/system.tsx +++ b/admin/inertia/pages/settings/system.tsx @@ -209,7 +209,7 @@ export default function SettingsPage(props: { type="warning" variant="bordered" title="GPU Not Accessible to AI Assistant" - message="Your system has an NVIDIA GPU, but the AI Assistant can't access it. AI is running on CPU only, which is significantly slower." + message={`Your system has ${info?.gpuHealth?.gpuVendor === 'amd' ? 'an AMD' : 'an NVIDIA'} GPU, but the AI Assistant can't access it. AI is running on CPU only, which is significantly slower.`} dismissible={true} onDismiss={handleDismissGpuBanner} buttonProps={{ diff --git a/admin/types/kv_store.ts b/admin/types/kv_store.ts index 8fb2686..a3632ab 100644 --- a/admin/types/kv_store.ts +++ b/admin/types/kv_store.ts @@ -12,6 +12,7 @@ export const KV_STORE_SCHEMA = { 'gpu.type': 'string', 'ai.remoteOllamaUrl': 'string', 'ai.ollamaFlashAttention': 'boolean', + 'ai.amdGpuAcceleration': 'boolean', } as const type KVTagToType = T extends 'boolean' ? boolean : string diff --git a/admin/types/system.ts b/admin/types/system.ts index 7c4e6d7..c8ed4ab 100644 --- a/admin/types/system.ts +++ b/admin/types/system.ts @@ -3,7 +3,9 @@ import { Systeminformation } from 'systeminformation' export type GpuHealthStatus = { status: 'ok' | 'passthrough_failed' | 'no_gpu' | 'ollama_not_installed' hasNvidiaRuntime: boolean + hasRocmRuntime: boolean ollamaGpuAccessible: boolean + gpuVendor?: 'nvidia' | 'amd' } export type SystemInformationResponse = { diff --git a/install/install_nomad.sh b/install/install_nomad.sh index ced178f..ef501a0 100644 --- a/install/install_nomad.sh +++ b/install/install_nomad.sh @@ -517,18 +517,35 @@ verify_gpu_setup() { echo -e "${YELLOW}○${RESET} Docker NVIDIA runtime not detected\\n" fi - # Check for AMD GPU + # Check for AMD GPU — restrict to display controller classes to avoid false positives + # from AMD CPU host bridges, PCI bridges, and chipset devices. + local has_amd_gpu='false' if command -v lspci &> /dev/null; then - if lspci 2>/dev/null | grep -iE "amd|radeon" &> /dev/null; then - echo -e "${YELLOW}○${RESET} AMD GPU detected (ROCm support not currently available)\\n" + if lspci 2>/dev/null | grep -iE "VGA|3D controller|Display" | grep -iE "amd|radeon" &> /dev/null; then + has_amd_gpu='true' + echo -e "${GREEN}✓${RESET} AMD GPU detected — ROCm acceleration will be configured automatically when AI Assistant is installed.\\n" fi fi - + + # Write detected GPU type to a marker file the admin container can read. The admin + # container lacks lspci and AMD GPUs don't register a Docker runtime, so this is the + # only reliable way for the admin to know an AMD GPU is present at install time. + local gpu_marker_path="${NOMAD_DIR}/storage/.nomad-gpu-type" + if command -v nvidia-smi &> /dev/null; then + echo 'nvidia' | sudo tee "${gpu_marker_path}" > /dev/null 2>&1 || true + elif [[ "${has_amd_gpu}" == 'true' ]]; then + echo 'amd' | sudo tee "${gpu_marker_path}" > /dev/null 2>&1 || true + else + sudo rm -f "${gpu_marker_path}" 2>/dev/null || true + fi + echo -e "${YELLOW}===========================================${RESET}\\n" - + # Summary if command -v nvidia-smi &> /dev/null && docker info 2>/dev/null | grep -q "nvidia"; then echo -e "${GREEN}#${RESET} GPU acceleration is properly configured! The AI Assistant will use your GPU.\\n" + elif [[ "${has_amd_gpu}" == 'true' ]]; then + echo -e "${GREEN}#${RESET} GPU acceleration will be enabled (AMD/ROCm) when AI Assistant is installed from the dashboard.\\n" else echo -e "${YELLOW}#${RESET} GPU acceleration not detected. The AI Assistant will run in CPU-only mode.\\n" if command -v nvidia-smi &> /dev/null && ! docker info 2>/dev/null | grep -q "nvidia"; then From 0836d84bb21843456e83ab8a59003d132ea77d0e Mon Sep 17 00:00:00 2001 From: Kenneth Brewer Date: Wed, 29 Apr 2026 00:55:11 -0400 Subject: [PATCH 03/18] docs: added notes field info to the map pin API reference (#803) --- admin/docs/api-reference.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/admin/docs/api-reference.md b/admin/docs/api-reference.md index 928b1e3..c14348c 100644 --- a/admin/docs/api-reference.md +++ b/admin/docs/api-reference.md @@ -153,8 +153,8 @@ ZIM files provide offline Wikipedia, books, and other content via Kiwix. | Method | Path | Description | |--------|------|-------------| | GET | `/api/maps/markers` | List map markers | -| POST | `/api/maps/markers` | Add map marker (body: {"name": "Test Marker", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) | -| PATCH | `/api/maps/markers/{id}` | Update a map marker (body: {"name": "Test Marker", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) fields that don't change can be omitted| +| POST | `/api/maps/markers` | Add map marker (body: {"name": "Test Marker", "notes": "Example note", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) | +| PATCH | `/api/maps/markers/{id}` | Update a map marker (body: {"name": "Test Marker", "notes": "Example note", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) fields that don't change can be omitted| | DELETE | `/api/maps/markers/{id}` | Delete a map marker | --- From bb1834a36438e545f522d965cf64855c8cb1e1ce Mon Sep 17 00:00:00 2001 From: cuyua9 <2114364329@qq.com> Date: Mon, 4 May 2026 03:49:06 +0800 Subject: [PATCH 04/18] fix(UI): wire map file delete confirmation to API (#732) Co-authored-by: cuyua9 --- admin/inertia/lib/api.ts | 9 +++++++++ admin/inertia/pages/settings/maps.tsx | 29 ++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 0df95ae..2449259 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -130,6 +130,15 @@ class API { })() } + async deleteMapRegionFile(filename: string): Promise<{ message: string }> { + return catchInternal(async () => { + const response = await this.client.delete<{ message: string }>( + `/maps/${encodeURIComponent(filename)}` + ) + return response.data + })() + } + async downloadRemoteZimFile( url: string, metadata?: { title: string; summary?: string; author?: string; size_bytes?: number } diff --git a/admin/inertia/pages/settings/maps.tsx b/admin/inertia/pages/settings/maps.tsx index da97bb1..47055fa 100644 --- a/admin/inertia/pages/settings/maps.tsx +++ b/admin/inertia/pages/settings/maps.tsx @@ -29,6 +29,7 @@ export default function MapsManager(props: { const { openModal, closeAllModals } = useModals() const { addNotification } = useNotifications() const [downloading, setDownloading] = useState(false) + const [deletingFileKey, setDeletingFileKey] = useState(null) const { data: curatedCollections } = useQuery({ queryKey: [CURATED_COLLECTIONS_KEY], @@ -120,18 +121,40 @@ export default function MapsManager(props: { } } + async function deleteFile(file: FileEntry) { + if (file.type !== 'file') return + + try { + setDeletingFileKey(file.key) + await api.deleteMapRegionFile(file.key) + addNotification({ + type: 'success', + message: `${file.name} has been deleted.`, + }) + closeAllModals() + router.reload({ only: ['maps'] }) + } catch (error) { + console.error('Error deleting map file:', error) + addNotification({ + type: 'error', + message: `Failed to delete ${file.name}. Please try again.`, + }) + } finally { + setDeletingFileKey(null) + } + } + async function confirmDeleteFile(file: FileEntry) { openModal( { - closeAllModals() - }} + onConfirm={() => deleteFile(file)} onCancel={closeAllModals} open={true} confirmText="Delete" cancelText="Cancel" confirmVariant="danger" + confirmLoading={file.type === 'file' && deletingFileKey === file.key} >

Are you sure you want to delete {file.name}? This action cannot be undone. From 360e7a0af48714db06792896ce302b85c0543213 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Wed, 22 Apr 2026 14:36:05 -0700 Subject: [PATCH 05/18] feat(content-updates): show size, surface downloads in Active Downloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Content Updates had three UX problems that compounded: 1. No size column, so users had to guess how big an update would be before clicking Update All. Upstream /api/v1/resources/check-updates doesn't return size, so CollectionUpdateService now enriches each update with a Content-Length HEAD request in parallel (5s timeout, non-fatal on failure — the row just renders an em-dash). 2. Small ZIM updates (1-8 MB) never appeared in Active Downloads. Two causes, both fixed: handleApply / handleApplyAll didn't invalidate the download-jobs query after dispatching, and useDownloads idled at 30s between polls — enough for a fast job to dispatch, download, and get cleaned up by removeOnComplete before the next refetch. 3. applyUpdate didn't forward title / totalBytes to RunDownloadJob, so any update that did briefly surface in Active Downloads had no label and no byte-count progress, just a filename and a percentage. It now passes both (matching zim_service's dispatch pattern). Also parallelized applyAllUpdates so dispatching five updates doesn't serialize five sequential BullMQ round-trips. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../app/services/collection_update_service.ts | 49 ++++++++++++++----- admin/app/validators/common.ts | 1 + admin/inertia/hooks/useDownloads.ts | 5 +- admin/inertia/pages/settings/update.tsx | 19 ++++++- admin/types/collections.ts | 1 + 5 files changed, 60 insertions(+), 15 deletions(-) diff --git a/admin/app/services/collection_update_service.ts b/admin/app/services/collection_update_service.ts index fee6c14..2e19cac 100644 --- a/admin/app/services/collection_update_service.ts +++ b/admin/app/services/collection_update_service.ts @@ -53,8 +53,10 @@ export class CollectionUpdateService { `[CollectionUpdateService] Update check complete: ${response.data.length} update(s) available` ) + const updates = await this.enrichWithSizes(response.data) + return { - updates: response.data, + updates, checked_at: new Date().toISOString(), } } catch (error) { @@ -105,6 +107,8 @@ export class CollectionUpdateService { update.resource_type === 'zim' ? ZIM_MIME_TYPES : PMTILES_MIME_TYPES, forceNew: true, filetype: update.resource_type, + title: update.resource_id, + totalBytes: update.size_bytes, resourceMetadata: { resource_id: update.resource_id, version: update.latest_version, @@ -126,21 +130,42 @@ export class CollectionUpdateService { async applyAllUpdates( updates: ResourceUpdateInfo[] ): Promise<{ results: Array<{ resource_id: string; success: boolean; jobId?: string; error?: string }> }> { - const results: Array<{ - resource_id: string - success: boolean - jobId?: string - error?: string - }> = [] - - for (const update of updates) { - const result = await this.applyUpdate(update) - results.push({ resource_id: update.resource_id, ...result }) - } + const results = await Promise.all( + updates.map(async (update) => { + const result = await this.applyUpdate(update) + return { resource_id: update.resource_id, ...result } + }) + ) return { results } } + /** + * Fetch Content-Length for each update URL in parallel. HEAD failures are non-fatal — + * the update row just renders without a size. Bounded to HEAD_TIMEOUT_MS so a slow + * mirror doesn't block the whole check. + */ + private async enrichWithSizes(updates: ResourceUpdateInfo[]): Promise { + const HEAD_TIMEOUT_MS = 5000 + + return await Promise.all( + updates.map(async (update) => { + if (update.size_bytes) return update // Trust upstream if it already gave us one + try { + const head = await axios.head(update.download_url, { + timeout: HEAD_TIMEOUT_MS, + maxRedirects: 5, + validateStatus: (s) => s >= 200 && s < 400, + }) + const len = Number(head.headers['content-length']) + return Number.isFinite(len) && len > 0 ? { ...update, size_bytes: len } : update + } catch { + return update + } + }) + ) + } + private buildFilename(update: ResourceUpdateInfo): string { if (update.resource_type === 'zim') { return `${update.resource_id}_${update.latest_version}.zim` diff --git a/admin/app/validators/common.ts b/admin/app/validators/common.ts index 8fe78bd..7065d4c 100644 --- a/admin/app/validators/common.ts +++ b/admin/app/validators/common.ts @@ -100,6 +100,7 @@ const resourceUpdateInfoBase = vine.object({ installed_version: vine.string().trim(), latest_version: vine.string().trim().minLength(1), download_url: vine.string().url({ require_tld: false }).trim(), + size_bytes: vine.number().positive().optional(), }) export const applyContentUpdateValidator = vine.compile(resourceUpdateInfoBase) diff --git a/admin/inertia/hooks/useDownloads.ts b/admin/inertia/hooks/useDownloads.ts index 3cdb859..b03399d 100644 --- a/admin/inertia/hooks/useDownloads.ts +++ b/admin/inertia/hooks/useDownloads.ts @@ -19,8 +19,9 @@ const useDownloads = (props: useDownloadsProps) => { queryFn: () => api.listDownloadJobs(props.filetype), refetchInterval: (query) => { const data = query.state.data - // Only poll when there are active downloads; otherwise use a slower interval - return data && data.length > 0 ? 2000 : 30000 + // Idle poll is kept tight so newly-dispatched jobs surface quickly — small ZIM + // updates can complete in ~2s, so a 30s idle interval almost always missed them. + return data && data.length > 0 ? 2000 : 3000 }, enabled: props.enabled ?? true, }) diff --git a/admin/inertia/pages/settings/update.tsx b/admin/inertia/pages/settings/update.tsx index 23527fb..348040d 100644 --- a/admin/inertia/pages/settings/update.tsx +++ b/admin/inertia/pages/settings/update.tsx @@ -12,9 +12,10 @@ import type { ContentUpdateCheckResult, ResourceUpdateInfo } from '../../../type import api from '~/lib/api' import Input from '~/components/inputs/Input' import Switch from '~/components/inputs/Switch' -import { useMutation } from '@tanstack/react-query' +import { useMutation, useQueryClient } from '@tanstack/react-query' import { useNotifications } from '~/context/NotificationContext' import { useSystemSetting } from '~/hooks/useSystemSetting' +import { formatBytes } from '~/lib/util' type Props = { updateAvailable: boolean @@ -25,6 +26,7 @@ type Props = { function ContentUpdatesSection() { const { addNotification } = useNotifications() + const queryClient = useQueryClient() const [checkResult, setCheckResult] = useState(null) const [isChecking, setIsChecking] = useState(false) const [applyingIds, setApplyingIds] = useState>(new Set()) @@ -60,6 +62,9 @@ function ContentUpdatesSection() { ? { ...prev, updates: prev.updates.filter((u) => u.resource_id !== update.resource_id) } : prev ) + // Force Active Downloads to refetch now — small updates finish before the next + // idle poll fires, so without this the user wouldn't see them. + queryClient.invalidateQueries({ queryKey: ['download-jobs'] }) } else { addNotification({ type: 'error', message: result?.error || 'Failed to start update' }) } @@ -95,6 +100,9 @@ function ContentUpdatesSection() { ? { ...prev, updates: prev.updates.filter((u) => !successIds.has(u.resource_id)) } : prev ) + if (successIds.size > 0) { + queryClient.invalidateQueries({ queryKey: ['download-jobs'] }) + } } } catch { addNotification({ type: 'error', message: 'Failed to apply updates' }) @@ -182,6 +190,15 @@ function ContentUpdatesSection() { ), }, + { + accessor: 'size_bytes', + title: 'Size', + render: (record) => ( + + {record.size_bytes ? formatBytes(record.size_bytes, 1) : '—'} + + ), + }, { accessor: 'installed_version', title: 'Version', diff --git a/admin/types/collections.ts b/admin/types/collections.ts index 1ec6d5c..abd47fc 100644 --- a/admin/types/collections.ts +++ b/admin/types/collections.ts @@ -86,6 +86,7 @@ export type ResourceUpdateInfo = { installed_version: string latest_version: string download_url: string + size_bytes?: number } export type ContentUpdateCheckResult = { From 27cd80309025dc63ab5ec4fdc86ef828b49272e1 Mon Sep 17 00:00:00 2001 From: 0xGlitch <92540908+bgauger@users.noreply.github.com> Date: Sun, 3 May 2026 14:47:53 -0600 Subject: [PATCH 06/18] feat(Maps): regional map downloads via go-pmtiles extract (#780) * feat(maps): add regional map downloads via go-pmtiles extract * address Copilot review feedback on PR #780 - auto-refresh preflight on selection/maxzoom change with 400ms debounce and requestId stale-safety so the confirm button no longer requires a two-step "Estimate Size" -> "Start Download" dance - safeUpdateProgress helper replaces fire-and-forget updateProgress().catch() pattern so cancelled-job errors (code -1) can't surface as unhandled rejections - gate world basemap source on worldBasemapReady - when ensureWorldBasemap() fails we already delete world.pmtiles, so emitting the source was producing 404s on every tile request - verify go-pmtiles binary SHA256 at image build time; upstream doesn't ship a checksums file so per-arch hashes are pinned as build args with a regenerate note when bumping PMTILES_VERSION --- Dockerfile | 25 ++ admin/adonisrc.ts | 4 + admin/app/controllers/maps_controller.ts | 24 ++ admin/app/jobs/run_extract_pmtiles_job.ts | 294 ++++++++++++++ admin/app/services/countries_service.ts | 308 ++++++++++++++ admin/app/services/download_service.ts | 112 +++-- admin/app/services/map_service.ts | 322 ++++++++++++++- admin/app/validators/common.ts | 28 ++ admin/commands/queue/work.ts | 6 + admin/constants/map_regions.ts | 32 ++ .../inertia/components/CountryPickerModal.tsx | 384 ++++++++++++++++++ admin/inertia/lib/api.ts | 41 ++ admin/inertia/pages/settings/maps.tsx | 33 ++ .../geodata/ne_50m_admin_0_countries.geojson | 1 + admin/start/routes.ts | 4 + admin/types/maps.ts | 34 ++ 16 files changed, 1620 insertions(+), 32 deletions(-) create mode 100644 admin/app/jobs/run_extract_pmtiles_job.ts create mode 100644 admin/app/services/countries_service.ts create mode 100644 admin/constants/map_regions.ts create mode 100644 admin/inertia/components/CountryPickerModal.tsx create mode 100644 admin/resources/geodata/ne_50m_admin_0_countries.geojson diff --git a/Dockerfile b/Dockerfile index 8850e23..0dcc184 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,31 @@ FROM base ARG VERSION=dev ARG BUILD_DATE ARG VCS_REF +ARG TARGETARCH + +# go-pmtiles (regional map extracts). Pinned so the CLI's stdout format stays +# in sync with parseDryRunOutput(). +ARG PMTILES_VERSION=1.30.2 +# Upstream releases don't ship a checksums file, so pin per-arch SHA256 here. +# When bumping PMTILES_VERSION, regenerate these with: +# curl -fsSL | sha256sum +ARG PMTILES_SHA256_AMD64=2cd3aa18868297fc88425038f794efdc0995e0275f4ca16fa496dd79e245a40c +ARG PMTILES_SHA256_ARM64=804cdf071834e1156af554c1a26cc42b56b9cde5a2db9c6e3653d16fb846d5fa +RUN set -eux; \ + case "${TARGETARCH:-amd64}" in \ + amd64) PMTILES_ARCH=x86_64; PMTILES_SHA256="${PMTILES_SHA256_AMD64}" ;; \ + arm64) PMTILES_ARCH=arm64; PMTILES_SHA256="${PMTILES_SHA256_ARM64}" ;; \ + *) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac; \ + TARBALL="go-pmtiles_${PMTILES_VERSION}_Linux_${PMTILES_ARCH}.tar.gz"; \ + cd /tmp; \ + curl -fsSL -o "$TARBALL" \ + "https://github.com/protomaps/go-pmtiles/releases/download/v${PMTILES_VERSION}/${TARBALL}"; \ + echo "${PMTILES_SHA256} ${TARBALL}" | sha256sum -c -; \ + tar -xzf "$TARBALL" -C /usr/local/bin pmtiles; \ + rm -f "$TARBALL"; \ + chmod +x /usr/local/bin/pmtiles; \ + /usr/local/bin/pmtiles version # Labels LABEL org.opencontainers.image.title="Project N.O.M.A.D" \ diff --git a/admin/adonisrc.ts b/admin/adonisrc.ts index a091ce2..34586ca 100644 --- a/admin/adonisrc.ts +++ b/admin/adonisrc.ts @@ -107,6 +107,10 @@ export default defineConfig({ pattern: 'resources/views/**/*.edge', reloadServer: false, }, + { + pattern: 'resources/geodata/**/*.geojson', + reloadServer: false, + }, { pattern: 'public/**', reloadServer: false, diff --git a/admin/app/controllers/maps_controller.ts b/admin/app/controllers/maps_controller.ts index dd93a8b..097d9fb 100644 --- a/admin/app/controllers/maps_controller.ts +++ b/admin/app/controllers/maps_controller.ts @@ -4,6 +4,8 @@ import { assertNotPrivateUrl, downloadCollectionValidator, filenameParamValidator, + mapExtractPreflightValidator, + mapExtractValidator, remoteDownloadValidator, remoteDownloadValidatorOptional, } from '#validators/common' @@ -87,6 +89,28 @@ export default class MapsController { } } + async listCountries({}: HttpContext) { + return { countries: await this.mapService.listCountries() } + } + + async listCountryGroups({}: HttpContext) { + return { groups: await this.mapService.listCountryGroups() } + } + + async extractPreflight({ request }: HttpContext) { + const payload = await request.validateUsing(mapExtractPreflightValidator) + return await this.mapService.extractPreflight(payload) + } + + async extractRegion({ request }: HttpContext) { + const payload = await request.validateUsing(mapExtractValidator) + const result = await this.mapService.extractRegion(payload) + return { + message: 'Extract started successfully', + ...result, + } + } + async styles({ request, response }: HttpContext) { // Automatically ensure base assets are present before generating styles const baseAssetsExist = await this.mapService.ensureBaseAssets() diff --git a/admin/app/jobs/run_extract_pmtiles_job.ts b/admin/app/jobs/run_extract_pmtiles_job.ts new file mode 100644 index 0000000..73c4eed --- /dev/null +++ b/admin/app/jobs/run_extract_pmtiles_job.ts @@ -0,0 +1,294 @@ +import { Job, UnrecoverableError } from 'bullmq' +import { spawn, ChildProcess } from 'child_process' +import { createHash } from 'crypto' +import { readdir, stat } from 'fs/promises' +import { basename, dirname, join } from 'path' +import { QueueService } from '#services/queue_service' +import logger from '@adonisjs/core/services/logger' +import { DownloadProgressData } from '../../types/downloads.js' +import { PMTILES_BINARY_PATH, buildPmtilesExtractArgs } from '../../constants/map_regions.js' +import { deleteFileIfExists } from '../utils/fs.js' + +export interface RunExtractPmtilesJobParams { + sourceUrl: string + outputFilepath: string + /** Path to a GeoJSON FeatureCollection file passed to `pmtiles extract --region`. */ + regionFilepath: string + maxzoom?: number + /** Hint for progress reporting; obtained from `pmtiles extract --dry-run` preflight */ + estimatedBytes?: number + filetype: 'map' + title?: string + resourceMetadata?: { + resource_id: string + version: string + collection_ref: string | null + } +} + +export class RunExtractPmtilesJob { + static get queue() { + return 'pmtiles-extract' + } + + static get key() { + return 'run-pmtiles-extract' + } + + /** In-memory registry of active child processes so in-process cancels can SIGTERM them */ + static childProcesses: Map = new Map() + + static getJobId(sourceUrl: string, regionFilepath: string, maxzoom?: number): string { + const payload = JSON.stringify({ sourceUrl, regionFilepath, maxzoom: maxzoom ?? null }) + return createHash('sha256').update(payload).digest('hex').slice(0, 16) + } + + /** Redis key used to signal cancellation across processes */ + static cancelKey(jobId: string): string { + return `nomad:download:pmtiles-cancel:${jobId}` + } + + static async signalCancel(jobId: string): Promise { + const queueService = new QueueService() + const queue = queueService.getQueue(this.queue) + const client = await queue.client + await client.set(this.cancelKey(jobId), '1', 'EX', 300) + } + + /** Awaits job.updateProgress and swallows BullMQ stale-job errors (code -1), + * which occur when the job was removed from Redis (e.g. cancelled) between + * the await being issued and the Redis write completing. Anything else + * re-throws so it's caught by the surrounding try rather than becoming an + * unhandled rejection. */ + private async safeUpdateProgress(job: Job, progress: DownloadProgressData): Promise { + try { + await job.updateProgress(progress) + } catch (err: any) { + if (err?.code !== -1) throw err + } + } + + async handle(job: Job) { + const params = job.data as RunExtractPmtilesJobParams + const { sourceUrl, outputFilepath, regionFilepath, maxzoom, estimatedBytes } = params + + logger.info( + `[RunExtractPmtilesJob] Starting extract: source=${sourceUrl} region=${regionFilepath} ` + + `maxzoom=${maxzoom ?? 'source-max'} out=${outputFilepath}` + ) + + const queueService = new QueueService() + const cancelRedis = await queueService.getQueue(RunExtractPmtilesJob.queue).client + + let userCancelled = false + let proc: ChildProcess | null = null + let lastReportedBytes = -1 + + // One 2s tick polls the Redis cancel signal and reads file-size for progress. pmtiles + // writes incrementally but rewrites directories near the end so progress isn't strictly + // monotonic — we cap at 99% and skip emit when bytes are unchanged to avoid Redis chatter. + const tick = setInterval(async () => { + try { + const val = await cancelRedis.get(RunExtractPmtilesJob.cancelKey(job.id!)) + if (val) { + await cancelRedis.del(RunExtractPmtilesJob.cancelKey(job.id!)) + userCancelled = true + proc?.kill('SIGTERM') + } + } catch { + // Redis errors non-fatal — in-memory handle also covers same-process cancels + } + + try { + const fileStat = await stat(outputFilepath) + const downloadedBytes = Number(fileStat.size) + if (downloadedBytes === lastReportedBytes) return + lastReportedBytes = downloadedBytes + + const totalBytes = estimatedBytes ?? 0 + const percent = + totalBytes > 0 ? Math.min(99, Math.floor((downloadedBytes / totalBytes) * 100)) : 0 + + await this.safeUpdateProgress(job, { + percent, + downloadedBytes, + totalBytes, + lastProgressTime: Date.now(), + } as DownloadProgressData) + } catch { + // File doesn't exist yet (subprocess still setting up) + } + }, 2000) + + try { + const args = buildPmtilesExtractArgs({ + sourceUrl, + outputFilepath, + regionFilepath, + maxzoom, + downloadThreads: 8, + overfetch: 0.2, + }) + proc = spawn(PMTILES_BINARY_PATH, args, { stdio: ['ignore', 'pipe', 'pipe'] }) + RunExtractPmtilesJob.childProcesses.set(job.id!, proc) + + proc.stdout?.on('data', (chunk) => { + logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`) + }) + proc.stderr?.on('data', (chunk) => { + logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`) + }) + + const exitCode: number = await new Promise((resolve, reject) => { + proc!.on('close', (code) => resolve(code ?? -1)) + proc!.on('error', (err) => reject(err)) + }) + + if (exitCode !== 0) { + await deleteFileIfExists(outputFilepath) + if (userCancelled) { + throw new UnrecoverableError(`Extract cancelled by user (exit ${exitCode})`) + } + throw new Error(`pmtiles extract exited with code ${exitCode}`) + } + + // Final progress bump — tick caps at 99 so the UI doesn't flicker to 100 mid-extract + const finalStat = await stat(outputFilepath) + await this.safeUpdateProgress(job, { + percent: 100, + downloadedBytes: Number(finalStat.size), + totalBytes: estimatedBytes ?? Number(finalStat.size), + lastProgressTime: Date.now(), + } as DownloadProgressData) + + // Reuse the HTTP download path's post-download hook so the file is registered and + // the previous version (if any) is deleted + await this.onComplete(params) + + logger.info( + `[RunExtractPmtilesJob] Completed extract: out=${outputFilepath} size=${finalStat.size} bytes` + ) + + return { sourceUrl, outputFilepath } + } catch (error: any) { + if (userCancelled && !(error instanceof UnrecoverableError)) { + throw new UnrecoverableError(`Extract cancelled: ${error.message ?? error}`) + } + throw error + } finally { + clearInterval(tick) + RunExtractPmtilesJob.childProcesses.delete(job.id!) + } + } + + private async onComplete(params: RunExtractPmtilesJobParams) { + if (!params.resourceMetadata) return + + const [{ default: InstalledResource }, { DateTime }, fsUtils] = await Promise.all([ + import('#models/installed_resource'), + import('luxon'), + import('../utils/fs.js'), + ]) + + const fileStat = await fsUtils.getFileStatsIfExists(params.outputFilepath) + + const existing = await InstalledResource.query() + .where('resource_id', params.resourceMetadata.resource_id) + .where('resource_type', 'map') + .first() + const oldFilePath = existing?.file_path ?? null + + await InstalledResource.updateOrCreate( + { + resource_id: params.resourceMetadata.resource_id, + resource_type: 'map', + }, + { + version: params.resourceMetadata.version, + collection_ref: params.resourceMetadata.collection_ref, + url: params.sourceUrl, + file_path: params.outputFilepath, + file_size_bytes: fileStat ? Number(fileStat.size) : null, + installed_at: DateTime.now(), + } + ) + + if (oldFilePath && oldFilePath !== params.outputFilepath) { + try { + await fsUtils.deleteFileIfExists(oldFilePath) + } catch (err) { + logger.warn(`[RunExtractPmtilesJob] Failed to delete old file ${oldFilePath}: ${err}`) + } + } + + // Fallback: scan the pmtiles dir for orphans with the same resource_id that the DB + // lookup above didn't catch — e.g. a prior extract crashed before writing its + // InstalledResource row, or an earlier bug wrote a file without registering it. + // Matches both curated (`_YYYY-MM.pmtiles`) and regional (`_YYYYMMDD_zN.pmtiles`) + // naming — prefix-only so new filename formats don't silently miss. + const dir = dirname(params.outputFilepath) + const keepName = basename(params.outputFilepath) + const prefix = `${params.resourceMetadata.resource_id}_` + try { + const entries = await readdir(dir) + for (const entry of entries) { + if (entry === keepName || !entry.endsWith('.pmtiles')) continue + if (!entry.startsWith(prefix)) continue + const orphanPath = join(dir, entry) + if (orphanPath === oldFilePath) continue + try { + await fsUtils.deleteFileIfExists(orphanPath) + logger.info(`[RunExtractPmtilesJob] Pruned orphan pmtiles ${orphanPath}`) + } catch (err) { + logger.warn(`[RunExtractPmtilesJob] Failed to prune orphan ${orphanPath}: ${err}`) + } + } + } catch (err) { + logger.warn(`[RunExtractPmtilesJob] Directory scan for orphans failed: ${err}`) + } + } + + static async getById(jobId: string): Promise { + const queueService = new QueueService() + const queue = queueService.getQueue(this.queue) + return await queue.getJob(jobId) + } + + static async dispatch(params: RunExtractPmtilesJobParams) { + const queueService = new QueueService() + const queue = queueService.getQueue(this.queue) + const jobId = this.getJobId(params.sourceUrl, params.regionFilepath, params.maxzoom) + + const existing = await queue.getJob(jobId) + if (existing) { + const state = await existing.getState() + if (state === 'active' || state === 'waiting' || state === 'delayed') { + return { + job: existing, + created: false, + message: `Extract job already exists for these params`, + } + } + // Stale (completed/failed) — remove so we can re-dispatch under the same deterministic id + try { + await existing.remove() + } catch { + // Already gone or locked — add() below will still report a meaningful error + } + } + + // Fewer attempts than HTTP downloads — a failed extract usually means the source URL + // rotated or the CDN is throttling, and resuming mid-extract isn't supported by the CLI + const job = await queue.add(this.key, params, { + jobId, + attempts: 3, + backoff: { type: 'exponential', delay: 60000 }, + removeOnComplete: true, + }) + return { + job, + created: true, + message: `Dispatched pmtiles extract job`, + } + } +} diff --git a/admin/app/services/countries_service.ts b/admin/app/services/countries_service.ts new file mode 100644 index 0000000..b6ebf0f --- /dev/null +++ b/admin/app/services/countries_service.ts @@ -0,0 +1,308 @@ +import { access, readFile, writeFile, mkdir } from 'fs/promises' +import { join, resolve } from 'path' +import { createHash } from 'crypto' +import { tmpdir } from 'os' +import logger from '@adonisjs/core/services/logger' +import type { Country, CountryCode, CountryGroup } from '../../types/maps.js' + +interface NEFeature { + type: 'Feature' + properties: Record + geometry: unknown +} + +interface NEFeatureCollection { + type: 'FeatureCollection' + features: NEFeature[] +} + +const COUNTRY_GEOJSON_PATH = join( + process.cwd(), + 'resources', + 'geodata', + 'ne_50m_admin_0_countries.geojson' +) + +// Natural Earth country polygons are land-only (no territorial waters), so a +// strict intersect leaves tiles fully over the ocean out of the extract — +// coastal cities render as grey off their coast. Inflate each polygon outward +// by ~11 km to pull in adjacent tiles without ballooning the extract size. +const REGION_BUFFER_DEGREES = 0.1 + +const GROUP_ORDER = [ + 'north-america', + 'south-america', + 'europe', + 'africa', + 'asia', + 'oceania', +] + +const GROUP_META: Record = { + 'North America': { + id: 'north-america', + name: 'North America', + description: 'All countries in North America and the Caribbean.', + }, + 'South America': { + id: 'south-america', + name: 'South America', + description: 'All countries in South America.', + }, + Europe: { + id: 'europe', + name: 'Europe', + description: 'All countries in Europe.', + }, + Africa: { + id: 'africa', + name: 'Africa', + description: 'All countries in Africa.', + }, + Asia: { + id: 'asia', + name: 'Asia', + description: 'All countries in Asia.', + }, + Oceania: { + id: 'oceania', + name: 'Oceania', + description: 'Australia, New Zealand, and Pacific island nations.', + }, +} + +export class CountriesService { + private static instance: CountriesService | null = null + private loadPromise: Promise | null = null + private countries: Country[] = [] + private byCode: Map = new Map() + private groups: CountryGroup[] = [] + + static getInstance(): CountriesService { + if (!this.instance) { + this.instance = new CountriesService() + } + return this.instance + } + + private async ensureLoaded(): Promise { + if (this.byCode.size > 0) return + if (!this.loadPromise) { + this.loadPromise = this.load() + } + await this.loadPromise + } + + private async load(): Promise { + const raw = await readFile(COUNTRY_GEOJSON_PATH, 'utf8') + const fc = JSON.parse(raw) as NEFeatureCollection + + // Natural Earth reuses a sovereign state's ISO_A2 for its dependencies + // (e.g. AU covers both Australia and Australian territories). Sort so the + // sovereign mainland wins the ISO-code slot, and skip any subsequent + // same-code dependency — otherwise the "AU" entry ends up being some tiny + // island territory. + const sortedFeatures = [...fc.features].sort((a, b) => typeRank(a) - typeRank(b)) + + const countries: Country[] = [] + const byCode = new Map() + const groupCodes: Record = {} + + for (const feature of sortedFeatures) { + const p = feature.properties + const code = resolveIso2(p) + if (!code) continue + if (byCode.has(code)) continue + + const continent = typeof p.CONTINENT === 'string' ? p.CONTINENT : 'Other' + if (continent === 'Antarctica' || continent === 'Seven seas (open ocean)') continue + + const country: Country = { + code, + code3: resolveIso3(p) ?? code, + name: typeof p.NAME === 'string' ? p.NAME : code, + continent, + subregion: typeof p.SUBREGION === 'string' ? p.SUBREGION : continent, + population: typeof p.POP_EST === 'number' ? p.POP_EST : 0, + } + + countries.push(country) + byCode.set(code, { country, feature }) + + if (GROUP_META[continent]) { + const groupId = GROUP_META[continent].id + if (!groupCodes[groupId]) groupCodes[groupId] = [] + groupCodes[groupId].push(code) + } + } + + countries.sort((a, b) => a.name.localeCompare(b.name)) + + const groups: CountryGroup[] = GROUP_ORDER.flatMap((groupId) => { + const meta = Object.values(GROUP_META).find((m) => m.id === groupId) + if (!meta) return [] + const codes = (groupCodes[groupId] ?? []).slice().sort() + if (codes.length === 0) return [] + return [{ id: meta.id, name: meta.name, description: meta.description, countries: codes }] + }) + + this.countries = countries + this.byCode = byCode + this.groups = groups + + logger.info( + `[CountriesService] Loaded ${countries.length} countries across ${groups.length} groups` + ) + } + + async list(): Promise { + await this.ensureLoaded() + return this.countries + } + + async listGroups(): Promise { + await this.ensureLoaded() + return this.groups + } + + /** Throws when a supplied code does not map to a known country. */ + async resolveCodes(codes: CountryCode[]): Promise { + await this.ensureLoaded() + const normalized = [...new Set(codes.map((c) => c.toUpperCase()))].sort() + const unknown = normalized.filter((c) => !this.byCode.has(c)) + if (unknown.length > 0) { + throw new Error(`Unknown country code(s): ${unknown.join(', ')}`) + } + return normalized + } + + /** + * Filename is keyed on a hash of the sorted ISO codes + buffer size so + * repeated calls with the same selection reuse the same path, and bumping + * the buffer auto-invalidates stale files. + */ + async writeRegionFile(codes: CountryCode[]): Promise { + await this.ensureLoaded() + const resolved = await this.resolveCodes(codes) + const key = `b${REGION_BUFFER_DEGREES}:${resolved.join(',')}` + const hash = createHash('sha1').update(key).digest('hex').slice(0, 12) + + const dir = resolve(tmpdir(), 'nomad-pmtiles-regions') + await mkdir(dir, { recursive: true }) + const filepath = join(dir, `region-${hash}.geojson`) + + try { + await access(filepath) + return filepath + } catch {} + + const fc = { + type: 'FeatureCollection', + features: resolved.map((code) => { + const entry = this.byCode.get(code)! + return { + type: 'Feature', + properties: { iso: code, name: entry.country.name }, + geometry: bufferGeometry(entry.feature.geometry, REGION_BUFFER_DEGREES), + } + }), + } + + await writeFile(filepath, JSON.stringify(fc)) + return filepath + } +} + +function typeRank(f: NEFeature): number { + const t = typeof f.properties.TYPE === 'string' ? f.properties.TYPE : '' + if (t === 'Sovereign country') return 0 + if (t === 'Country') return 1 + if (t === 'Sovereignty') return 2 + if (t === 'Disputed') return 3 + if (t === 'Dependency') return 4 + return 5 +} + +function resolveIso2(p: Record): CountryCode | null { + // Natural Earth's ISO_A2 sometimes holds political escapes like "CN-TW" for + // Taiwan or "-99" for countries involved in disputes. Only accept clean + // 2-letter codes; fall back to ISO_A2_EH (which reliably has the real code). + const primary = typeof p.ISO_A2 === 'string' ? p.ISO_A2 : null + if (primary && /^[A-Z]{2}$/i.test(primary)) return primary.toUpperCase() + const fallback = typeof p.ISO_A2_EH === 'string' ? p.ISO_A2_EH : null + if (fallback && /^[A-Z]{2}$/i.test(fallback)) return fallback.toUpperCase() + return null +} + +/** + * Inflate each polygon ring outward by `buffer` degrees via per-vertex + * averaged-normal offset. Not geodesically accurate — but at small buffers + * (<= 0.2°) it's within a few percent of a proper geodesic buffer at + * country scale, which is plenty for tile-inclusion purposes. + */ +function bufferGeometry(geometry: unknown, buffer: number): unknown { + const geom = geometry as { type: string; coordinates: any } + if (geom?.type === 'Polygon') { + return { type: 'Polygon', coordinates: bufferPolygonRings(geom.coordinates, buffer) } + } + if (geom?.type === 'MultiPolygon') { + return { + type: 'MultiPolygon', + coordinates: geom.coordinates.map((poly: number[][][]) => + bufferPolygonRings(poly, buffer) + ), + } + } + return geometry +} + +function bufferPolygonRings(rings: number[][][], buffer: number): number[][][] { + return rings.map((ring) => bufferRing(ring, buffer)) +} + +function bufferRing(ring: number[][], buffer: number): number[][] { + if (ring.length < 4) return ring + const sign = signedArea(ring) > 0 ? 1 : -1 + const n = ring.length - 1 + const out: number[][] = [] + for (let i = 0; i < n; i++) { + const prev = ring[(i - 1 + n) % n] + const curr = ring[i] + const next = ring[(i + 1) % n] + const e1x = curr[0] - prev[0] + const e1y = curr[1] - prev[1] + const e2x = next[0] - curr[0] + const e2y = next[1] - curr[1] + const l1 = Math.hypot(e1x, e1y) || 1 + const l2 = Math.hypot(e2x, e2y) || 1 + const n1x = (e1y / l1) * sign + const n1y = (-e1x / l1) * sign + const n2x = (e2y / l2) * sign + const n2y = (-e2x / l2) * sign + const sumX = n1x + n2x + const sumY = n1y + n2y + const sl = Math.hypot(sumX, sumY) || 1 + out.push([curr[0] + (sumX / sl) * buffer, curr[1] + (sumY / sl) * buffer]) + } + out.push(out[0]) + return out +} + +function signedArea(ring: number[][]): number { + let a = 0 + for (let i = 0; i < ring.length - 1; i++) { + a += ring[i][0] * ring[i + 1][1] - ring[i + 1][0] * ring[i][1] + } + return a / 2 +} + +function resolveIso3(p: Record): string | null { + const primary = typeof p.ISO_A3 === 'string' ? p.ISO_A3 : null + if (primary && primary !== '-99') return primary.toUpperCase() + const fallback = typeof p.ISO_A3_EH === 'string' ? p.ISO_A3_EH : null + if (fallback && fallback !== '-99') return fallback.toUpperCase() + const adm = typeof p.ADM0_A3 === 'string' ? p.ADM0_A3 : null + if (adm && adm !== '-99') return adm.toUpperCase() + return null +} + diff --git a/admin/app/services/download_service.ts b/admin/app/services/download_service.ts index bd9076c..91b8288 100644 --- a/admin/app/services/download_service.ts +++ b/admin/app/services/download_service.ts @@ -1,13 +1,19 @@ import { inject } from '@adonisjs/core' import { QueueService } from './queue_service.js' import { RunDownloadJob } from '#jobs/run_download_job' +import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job' +import type { RunExtractPmtilesJobParams } from '#jobs/run_extract_pmtiles_job' import { DownloadModelJob } from '#jobs/download_model_job' import { DownloadJobWithProgress, DownloadProgressData } from '../../types/downloads.js' +import type { Job, Queue } from 'bullmq' import { normalize } from 'path' import { deleteFileIfExists } from '../utils/fs.js' import transmit from '@adonisjs/transmit/services/main' import { BROADCAST_CHANNELS } from '../../constants/broadcast.js' +type FileJobState = 'waiting' | 'active' | 'delayed' | 'failed' +type TaggedJob = { job: Job; state: FileJobState } + @inject() export class DownloadService { constructor(private queueService: QueueService) {} @@ -26,27 +32,32 @@ export class DownloadService { return { percent: parseInt(String(progress), 10) || 0 } } - async listDownloadJobs(filetype?: string): Promise { - // Get regular file download jobs (zim, map, etc.) — query each state separately so we can - // tag each job with its actual BullMQ state rather than guessing from progress data. - const queue = this.queueService.getQueue(RunDownloadJob.queue) - type FileJobState = 'waiting' | 'active' | 'delayed' | 'failed' - - const [waitingJobs, activeJobs, delayedJobs, failedJobs] = await Promise.all([ + /** Fetch all non-completed jobs from a queue, tagged with their current BullMQ state */ + private async fetchJobsWithStates(queueName: string): Promise { + const queue = this.queueService.getQueue(queueName) + const [waiting, active, delayed, failed] = await Promise.all([ queue.getJobs(['waiting']), queue.getJobs(['active']), queue.getJobs(['delayed']), queue.getJobs(['failed']), ]) - - const taggedFileJobs: Array<{ job: (typeof waitingJobs)[0]; state: FileJobState }> = [ - ...waitingJobs.map((j) => ({ job: j, state: 'waiting' as const })), - ...activeJobs.map((j) => ({ job: j, state: 'active' as const })), - ...delayedJobs.map((j) => ({ job: j, state: 'delayed' as const })), - ...failedJobs.map((j) => ({ job: j, state: 'failed' as const })), + return [ + ...waiting.map((j) => ({ job: j, state: 'waiting' as const })), + ...active.map((j) => ({ job: j, state: 'active' as const })), + ...delayed.map((j) => ({ job: j, state: 'delayed' as const })), + ...failed.map((j) => ({ job: j, state: 'failed' as const })), ] + } - const fileDownloads = taggedFileJobs.map(({ job, state }) => { + async listDownloadJobs(filetype?: string): Promise { + const modelQueue = this.queueService.getQueue(DownloadModelJob.queue) + const [fileTagged, extractTagged, modelJobs] = await Promise.all([ + this.fetchJobsWithStates(RunDownloadJob.queue), + this.fetchJobsWithStates(RunExtractPmtilesJob.queue), + modelQueue.getJobs(['waiting', 'active', 'delayed', 'failed']), + ]) + + const fileDownloads = fileTagged.map(({ job, state }) => { const parsed = this.parseProgress(job.progress) return { jobId: job.id!.toString(), @@ -63,26 +74,36 @@ export class DownloadService { } }) - // Get Ollama model download jobs - const modelQueue = this.queueService.getQueue(DownloadModelJob.queue) - const modelJobs = await modelQueue.getJobs(['waiting', 'active', 'delayed', 'failed']) + const extractDownloads = extractTagged.map(({ job, state }) => { + const parsed = this.parseProgress(job.progress) + return { + jobId: job.id!.toString(), + url: job.data.sourceUrl, + progress: parsed.percent, + filepath: normalize(job.data.outputFilepath), + filetype: job.data.filetype || 'map', + title: job.data.title || undefined, + downloadedBytes: parsed.downloadedBytes, + totalBytes: parsed.totalBytes || job.data.estimatedBytes || undefined, + lastProgressTime: parsed.lastProgressTime, + status: state, + failedReason: job.failedReason || undefined, + } + }) const modelDownloads = modelJobs.map((job) => ({ jobId: job.id!.toString(), - url: job.data.modelName || 'Unknown Model', // Use model name as url + url: job.data.modelName || 'Unknown Model', progress: parseInt(job.progress.toString(), 10), - filepath: job.data.modelName || 'Unknown Model', // Use model name as filepath + filepath: job.data.modelName || 'Unknown Model', filetype: 'model', status: (job.failedReason ? 'failed' : 'active') as 'active' | 'failed', failedReason: job.failedReason || undefined, })) - const allDownloads = [...fileDownloads, ...modelDownloads] - - // Filter by filetype if specified + const allDownloads = [...fileDownloads, ...extractDownloads, ...modelDownloads] const filtered = allDownloads.filter((job) => !filetype || job.filetype === filetype) - // Sort: active downloads first (by progress desc), then failed at the bottom return filtered.sort((a, b) => { if (a.status === 'failed' && b.status !== 'failed') return 1 if (a.status !== 'failed' && b.status === 'failed') return -1 @@ -91,7 +112,11 @@ export class DownloadService { } async removeFailedJob(jobId: string): Promise { - for (const queueName of [RunDownloadJob.queue, DownloadModelJob.queue]) { + for (const queueName of [ + RunDownloadJob.queue, + RunExtractPmtilesJob.queue, + DownloadModelJob.queue, + ]) { const queue = this.queueService.getQueue(queueName) const job = await queue.getJob(jobId) if (job) { @@ -113,7 +138,6 @@ export class DownloadService { } async cancelJob(jobId: string): Promise<{ success: boolean; message: string }> { - // Try the file download queue first (the original PR #554 path) const queue = this.queueService.getQueue(RunDownloadJob.queue) const job = await queue.getJob(jobId) @@ -121,7 +145,13 @@ export class DownloadService { return await this._cancelFileDownloadJob(jobId, job, queue) } - // Fall through to the model download queue + const extractQueue = this.queueService.getQueue(RunExtractPmtilesJob.queue) + const extractJob = await extractQueue.getJob(jobId) + + if (extractJob) { + return await this._cancelExtractJob(jobId, extractJob, extractQueue) + } + const modelQueue = this.queueService.getQueue(DownloadModelJob.queue) const modelJob = await modelQueue.getJob(jobId) @@ -129,11 +159,37 @@ export class DownloadService { return await this._cancelModelDownloadJob(jobId, modelJob, modelQueue) } - // Not found in either queue return { success: true, message: 'Job not found (may have already completed)' } } - /** Cancel a content download (zim, map, pmtiles, etc.) — original PR #554 logic */ + private async _cancelExtractJob( + jobId: string, + job: Job, + queue: Queue + ): Promise<{ success: boolean; message: string }> { + const outputFilepath = job.data.outputFilepath + + await RunExtractPmtilesJob.signalCancel(jobId) + + // Same-process fallback when worker and API share a process + RunExtractPmtilesJob.childProcesses.get(jobId)?.kill('SIGTERM') + RunExtractPmtilesJob.childProcesses.delete(jobId) + + await this._pollForTerminalState(job, jobId) + await this._removeJobWithLockFallback(job, queue, RunExtractPmtilesJob.queue, jobId) + + if (outputFilepath) { + try { + await deleteFileIfExists(outputFilepath) + } catch { + // File may not exist yet (subprocess may not have opened it) + } + } + + return { success: true, message: 'Extract cancelled and partial file deleted' } + } + + /** Cancel a content download (zim, map, pmtiles, etc.) */ private async _cancelFileDownloadJob( jobId: string, job: any, diff --git a/admin/app/services/map_service.ts b/admin/app/services/map_service.ts index c9902a3..c1157b0 100644 --- a/admin/app/services/map_service.ts +++ b/admin/app/services/map_service.ts @@ -16,11 +16,35 @@ import { import { join, resolve, sep } from 'path' import urlJoin from 'url-join' import { RunDownloadJob } from '#jobs/run_download_job' +import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job' import logger from '@adonisjs/core/services/logger' import { assertNotPrivateUrl } from '#validators/common' import InstalledResource from '#models/installed_resource' import { CollectionManifestService } from './collection_manifest_service.js' import type { CollectionWithStatus, MapsSpec } from '../../types/collections.js' +import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps.js' +import { + EXTRACT_DEFAULT_MAX_ZOOM, + EXTRACT_MAX_ZOOM, + EXTRACT_MIN_ZOOM, + PMTILES_BINARY_PATH, + WORLD_BASEMAP_FILENAME, + WORLD_BASEMAP_MAX_ZOOM, + WORLD_BASEMAP_SOURCE_NAME, + buildPmtilesExtractArgs, +} from '../../constants/map_regions.js' +import { CountriesService } from './countries_service.js' +import { execFile } from 'child_process' +import { createHash, randomBytes } from 'crypto' +import { tmpdir } from 'os' +import { promisify } from 'util' + +const execFileAsync = promisify(execFile) +const DRY_RUN_TIMEOUT_MS = 60_000 +const DRY_RUN_MAX_BUFFER = 256 * 1024 +// Real extract of z0-5 world tiles; generous to tolerate slow/metered links +// since a failure leaves the map grey for uncovered regions. +const WORLD_BASEMAP_EXTRACT_TIMEOUT_MS = 5 * 60_000 const PROTOMAPS_BUILDS_METADATA_URL = 'https://build-metadata.protomaps.dev/builds.json' const PROTOMAPS_BUILD_BASE_URL = 'https://build.protomaps.com' @@ -53,10 +77,15 @@ export class MapService implements IMapService { private readonly baseAssetsTarFile = 'base-assets.tar.gz' private readonly baseDirPath = join(process.cwd(), this.mapStoragePath) private baseAssetsExistCache: boolean | null = null + private worldBasemapReady = false + private worldBasemapInFlight: Promise | null = null async listRegions() { const files = (await this.listAllMapStorageItems()).filter( - (item) => item.type === 'file' && item.name.endsWith('.pmtiles') + (item) => + item.type === 'file' && + item.name.endsWith('.pmtiles') && + item.name !== WORLD_BASEMAP_FILENAME ) return { @@ -327,11 +356,76 @@ export class MapService implements IMapService { async ensureBaseAssets(): Promise { const exists = await this.checkBaseAssetsExist() - if (exists) { - return true + if (!exists) { + const downloaded = await this.downloadBaseAssets() + if (!downloaded) return false } - return await this.downloadBaseAssets() + try { + await this.ensureWorldBasemap() + } catch (err) { + logger.warn(`[MapService] World basemap setup failed, continuing without it: ${err}`) + } + + return true + } + + /** + * Extract a low-zoom global basemap once so the map isn't grey outside a + * regional extract's polygon. Cheap (~15 MB, a handful of HTTP range + * requests) and layered underneath regional sources at render time. + * + * Memoizes success in-process, and de-duplicates concurrent callers via a + * shared in-flight promise so two simultaneous `/maps` requests on a cold + * start don't both launch `pmtiles extract` against the same output path. + */ + private async ensureWorldBasemap(): Promise { + if (this.worldBasemapReady) return + if (this.worldBasemapInFlight) return this.worldBasemapInFlight + this.worldBasemapInFlight = this._setupWorldBasemap().finally(() => { + this.worldBasemapInFlight = null + }) + return this.worldBasemapInFlight + } + + private async _setupWorldBasemap(): Promise { + const basePath = resolve(join(this.baseDirPath, 'pmtiles')) + const filepath = resolve(join(basePath, WORLD_BASEMAP_FILENAME)) + if (!filepath.startsWith(basePath + sep)) { + throw new Error('Invalid world basemap path') + } + + await ensureDirectoryExists(basePath) + + const existing = await getFileStatsIfExists(filepath) + if (existing && Number(existing.size) > 0) { + this.worldBasemapReady = true + return + } + + const info = await this.getGlobalMapInfo() + const args = buildPmtilesExtractArgs({ + sourceUrl: info.url, + outputFilepath: filepath, + maxzoom: WORLD_BASEMAP_MAX_ZOOM, + downloadThreads: 4, + }) + + logger.info( + `[MapService] Extracting world basemap (z0-${WORLD_BASEMAP_MAX_ZOOM}) from ${info.url}` + ) + try { + await execFileAsync(PMTILES_BINARY_PATH, args, { + timeout: WORLD_BASEMAP_EXTRACT_TIMEOUT_MS, + maxBuffer: DRY_RUN_MAX_BUFFER, + }) + this.worldBasemapReady = true + } catch (err: any) { + await deleteFileIfExists(filepath) + throw new Error( + `pmtiles extract for world basemap failed: ${err.message}. stderr: ${err.stderr ?? ''}` + ) + } } private async checkBaseAssetsExist(useCache: boolean = true): Promise { @@ -367,6 +461,19 @@ export class MapService implements IMapService { const sources: BaseStylesFile['sources'][] = [] const baseUrl = this.getPublicFileBaseUrl(host, 'pmtiles', protocol) + // World basemap goes first so its layers render underneath regional extracts. + // Only emitted when ensureWorldBasemap() succeeded — otherwise the style would + // reference a file that doesn't exist and produce 404s on every tile request. + if (this.worldBasemapReady) { + const worldSource: BaseStylesFile['sources'] = {} + worldSource[WORLD_BASEMAP_SOURCE_NAME] = { + type: 'vector', + attribution: PMTILES_ATTRIBUTION, + url: `pmtiles://${urlJoin(baseUrl, WORLD_BASEMAP_FILENAME)}`, + } + sources.push(worldSource) + } + for (const region of regions) { if (region.type === 'file' && region.name.endsWith('.pmtiles')) { // Strip .pmtiles and date suffix (e.g. "alaska_2025-12" -> "alaska") for stable source names @@ -489,12 +596,206 @@ export class MapService implements IMapService { } } + async listCountries(): Promise { + return CountriesService.getInstance().list() + } + + async listCountryGroups(): Promise { + return CountriesService.getInstance().listGroups() + } + + async extractPreflight(params: { + countries: CountryCode[] + maxzoom?: number + }): Promise { + this.validateMaxzoom(params.maxzoom) + const countries = await CountriesService.getInstance().resolveCodes(params.countries) + const regionFilepath = await CountriesService.getInstance().writeRegionFile(countries) + const info = await this.getGlobalMapInfo() + return this.runDryRun(info, regionFilepath, params.maxzoom) + } + + private async runDryRun( + info: { url: string; date: string; key: string }, + regionFilepath: string, + maxzoom?: number + ): Promise { + const dryRunOutput = join(tmpdir(), `pmtiles-dry-run-${randomBytes(6).toString('hex')}.pmtiles`) + const args = buildPmtilesExtractArgs({ + sourceUrl: info.url, + outputFilepath: dryRunOutput, + regionFilepath, + maxzoom, + dryRun: true, + }) + + let stdout = '' + let stderr = '' + try { + const result = await execFileAsync(PMTILES_BINARY_PATH, args, { + timeout: DRY_RUN_TIMEOUT_MS, + maxBuffer: DRY_RUN_MAX_BUFFER, + }) + stdout = result.stdout + stderr = result.stderr + } catch (err: any) { + throw new Error( + `pmtiles extract --dry-run failed: ${err.message}. stderr: ${err.stderr ?? ''}` + ) + } + + const parsed = this.parseDryRunOutput(stdout + '\n' + stderr) + + return { + tiles: parsed.tiles, + bytes: parsed.bytes, + source: { url: info.url, date: info.date, key: info.key }, + } + } + + async extractRegion(params: { + countries: CountryCode[] + maxzoom?: number + label?: string + estimatedBytes?: number + }): Promise<{ filename: string; jobId?: string }> { + this.validateMaxzoom(params.maxzoom) + const countriesService = CountriesService.getInstance() + const countries = await countriesService.resolveCodes(params.countries) + const regionFilepath = await countriesService.writeRegionFile(countries) + const maxzoom = params.maxzoom ?? EXTRACT_DEFAULT_MAX_ZOOM + + const [baseAssetsExist, info, groups] = await Promise.all([ + this.ensureBaseAssets(), + this.getGlobalMapInfo(), + countriesService.listGroups(), + ]) + if (!baseAssetsExist) { + throw new Error( + 'Base map assets are missing and could not be downloaded. Please check your connection and try again.' + ) + } + + const groupMatch = findExactGroupMatch(countries, groups) + const slug = this.buildRegionSlug(countries, groupMatch) + const dateSlug = info.key.replace('.pmtiles', '') + const filename = `${slug}_${dateSlug}_z${maxzoom}.pmtiles` + const basePath = resolve(join(this.baseDirPath, 'pmtiles')) + const filepath = resolve(join(basePath, filename)) + + if (!filepath.startsWith(basePath + sep)) { + throw new Error('Invalid filename') + } + + let estimatedBytes = params.estimatedBytes ?? 0 + if (estimatedBytes === 0) { + try { + const preflight = await this.runDryRun(info, regionFilepath, maxzoom) + estimatedBytes = preflight.bytes + } catch (err) { + logger.warn(`[MapService] extractRegion preflight failed, proceeding without estimate: ${err}`) + } + } + + const title = params.label ?? this.buildRegionTitle(countries, groupMatch) + + const result = await RunExtractPmtilesJob.dispatch({ + sourceUrl: info.url, + outputFilepath: filepath, + regionFilepath, + maxzoom, + estimatedBytes, + filetype: 'map', + title, + resourceMetadata: { + resource_id: slug, + version: dateSlug, + collection_ref: null, + }, + }) + + if (!result.job) { + throw new Error('Failed to dispatch extract job') + } + + logger.info( + `[MapService] Dispatched extract job ${result.job.id} for ${filename} ` + + `(countries=[${countries.join(',')}] maxzoom=${maxzoom} est=${estimatedBytes} bytes)` + ) + + return { + filename, + jobId: result.job.id, + } + } + + private buildRegionSlug(countries: CountryCode[], groupMatch: CountryGroup | null): string { + if (groupMatch) return groupMatch.id + if (countries.length === 1) return countries[0].toLowerCase() + const hash = createHash('sha1').update(countries.join(',')).digest('hex').slice(0, 8) + return `custom-${hash}` + } + + private buildRegionTitle(countries: CountryCode[], groupMatch: CountryGroup | null): string { + if (groupMatch) return groupMatch.name + if (countries.length === 1) return countries[0] + if (countries.length <= 3) return countries.join(', ') + return `${countries.slice(0, 2).join(', ')} +${countries.length - 2} more` + } + + private validateMaxzoom(maxzoom: number | undefined): void { + if (typeof maxzoom !== 'number') return + if ( + !Number.isInteger(maxzoom) || + maxzoom < EXTRACT_MIN_ZOOM || + maxzoom > EXTRACT_MAX_ZOOM + ) { + throw new Error( + `maxzoom must be an integer in [${EXTRACT_MIN_ZOOM}, ${EXTRACT_MAX_ZOOM}]` + ) + } + } + + // go-pmtiles output format isn't stable across versions — parse loosely and + // fall back to zeros. The extract can still proceed without an estimate. + private parseDryRunOutput(output: string): { tiles: number; bytes: number } { + let bytes = 0 + let tiles = 0 + + const byteLine = output.match(/archive\s+size\s+of\s+([\d,.]+)\s*(B|KB|MB|GB|TB|bytes?)?/i) + if (byteLine) { + const raw = parseFloat(byteLine[1].replace(/,/g, '')) + const unit = (byteLine[2] ?? 'B').toUpperCase() + const multipliers: Record = { + B: 1, + BYTE: 1, + BYTES: 1, + KB: 1_000, + MB: 1_000_000, + GB: 1_000_000_000, + TB: 1_000_000_000_000, + } + bytes = Math.round(raw * (multipliers[unit] ?? 1)) + } + + const tileLine = output.match(/(?:tiles\s+to\s+extract|tiles)[^\d]*([\d,]+)/i) + if (tileLine) { + tiles = parseInt(tileLine[1].replace(/,/g, ''), 10) || 0 + } + + return { tiles, bytes } + } + async delete(file: string): Promise { let fileName = file if (!fileName.endsWith('.pmtiles')) { fileName += '.pmtiles' } + if (fileName === WORLD_BASEMAP_FILENAME) { + throw new Error('The world basemap cannot be deleted') + } + const basePath = resolve(join(this.baseDirPath, 'pmtiles')) const fullPath = resolve(join(basePath, fileName)) @@ -573,3 +874,16 @@ export class MapService implements IMapService { return baseUrl } } + +function findExactGroupMatch( + countries: CountryCode[], + groups: CountryGroup[] +): CountryGroup | null { + return ( + groups.find( + (g) => + g.countries.length === countries.length && + g.countries.every((c, i) => c === countries[i]) + ) ?? null + ) +} diff --git a/admin/app/validators/common.ts b/admin/app/validators/common.ts index 7065d4c..ba9f107 100644 --- a/admin/app/validators/common.ts +++ b/admin/app/validators/common.ts @@ -112,3 +112,31 @@ export const applyAllContentUpdatesValidator = vine.compile( .minLength(1), }) ) + +// --- Map extract (regional pmtiles download) --- + +// ISO 3166-1 alpha-2, 2 letters. Loose regex; CountriesService.resolveCodes +// does the authoritative check against the polygon dataset. +const countryCodeSchema = vine + .string() + .trim() + .toUpperCase() + .regex(/^[A-Z]{2}$/) + +const countriesArraySchema = vine.array(countryCodeSchema).minLength(1).maxLength(300) + +export const mapExtractPreflightValidator = vine.compile( + vine.object({ + countries: countriesArraySchema.clone(), + maxzoom: vine.number().min(0).max(15).optional(), + }) +) + +export const mapExtractValidator = vine.compile( + vine.object({ + countries: countriesArraySchema.clone(), + maxzoom: vine.number().min(0).max(15).optional(), + label: vine.string().trim().minLength(1).maxLength(64).optional(), + estimatedBytes: vine.number().min(0).optional(), + }) +) diff --git a/admin/commands/queue/work.ts b/admin/commands/queue/work.ts index 31bb1cc..49890e6 100644 --- a/admin/commands/queue/work.ts +++ b/admin/commands/queue/work.ts @@ -3,6 +3,7 @@ import type { CommandOptions } from '@adonisjs/core/types/ace' import { Worker } from 'bullmq' import queueConfig from '#config/queue' import { RunDownloadJob } from '#jobs/run_download_job' +import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job' import { DownloadModelJob } from '#jobs/download_model_job' import { RunBenchmarkJob } from '#jobs/run_benchmark_job' import { EmbedFileJob } from '#jobs/embed_file_job' @@ -126,6 +127,7 @@ export default class QueueWork extends BaseCommand { const queues = new Map() handlers.set(RunDownloadJob.key, new RunDownloadJob()) + handlers.set(RunExtractPmtilesJob.key, new RunExtractPmtilesJob()) handlers.set(DownloadModelJob.key, new DownloadModelJob()) handlers.set(RunBenchmarkJob.key, new RunBenchmarkJob()) handlers.set(EmbedFileJob.key, new EmbedFileJob()) @@ -133,6 +135,7 @@ export default class QueueWork extends BaseCommand { handlers.set(CheckServiceUpdatesJob.key, new CheckServiceUpdatesJob()) queues.set(RunDownloadJob.key, RunDownloadJob.queue) + queues.set(RunExtractPmtilesJob.key, RunExtractPmtilesJob.queue) queues.set(DownloadModelJob.key, DownloadModelJob.queue) queues.set(RunBenchmarkJob.key, RunBenchmarkJob.queue) queues.set(EmbedFileJob.key, EmbedFileJob.queue) @@ -149,6 +152,9 @@ export default class QueueWork extends BaseCommand { private getConcurrencyForQueue(queueName: string): number { const concurrencyMap: Record = { [RunDownloadJob.queue]: 3, + // pmtiles extract hits the Protomaps CDN with many parallel range reads per job; + // cap concurrency at 2 so a second extract doesn't starve the first. + [RunExtractPmtilesJob.queue]: 2, [DownloadModelJob.queue]: 2, // Lower concurrency for resource-intensive model downloads [RunBenchmarkJob.queue]: 1, // Run benchmarks one at a time for accurate results [EmbedFileJob.queue]: 2, // Lower concurrency for embedding jobs, can be resource intensive diff --git a/admin/constants/map_regions.ts b/admin/constants/map_regions.ts new file mode 100644 index 0000000..75f2922 --- /dev/null +++ b/admin/constants/map_regions.ts @@ -0,0 +1,32 @@ +export const PMTILES_BINARY_PATH = '/usr/local/bin/pmtiles' + +// Clamp these so a user can't ask for nonsense that never extracts +export const EXTRACT_MIN_ZOOM = 0 +export const EXTRACT_MAX_ZOOM = 15 +export const EXTRACT_DEFAULT_MAX_ZOOM = 15 + +// Low-zoom global fallback extracted once during base-asset setup (~15 MB). Layered +// underneath regional extracts so the map isn't grey outside a region's polygon. +export const WORLD_BASEMAP_FILENAME = 'world.pmtiles' +export const WORLD_BASEMAP_MAX_ZOOM = 5 +export const WORLD_BASEMAP_SOURCE_NAME = 'world' + +export interface PmtilesExtractArgOptions { + sourceUrl: string + outputFilepath: string + regionFilepath?: string + maxzoom?: number + dryRun?: boolean + downloadThreads?: number + overfetch?: number +} + +export function buildPmtilesExtractArgs(opts: PmtilesExtractArgOptions): string[] { + const args = ['extract', opts.sourceUrl, opts.outputFilepath] + if (opts.regionFilepath) args.push(`--region=${opts.regionFilepath}`) + if (typeof opts.maxzoom === 'number') args.push(`--maxzoom=${opts.maxzoom}`) + if (opts.dryRun) args.push('--dry-run') + if (typeof opts.downloadThreads === 'number') args.push(`--download-threads=${opts.downloadThreads}`) + if (typeof opts.overfetch === 'number') args.push(`--overfetch=${opts.overfetch}`) + return args +} diff --git a/admin/inertia/components/CountryPickerModal.tsx b/admin/inertia/components/CountryPickerModal.tsx new file mode 100644 index 0000000..b490360 --- /dev/null +++ b/admin/inertia/components/CountryPickerModal.tsx @@ -0,0 +1,384 @@ +import { useEffect, useMemo, useRef, useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { IconCheck, IconSearch, IconX } from '@tabler/icons-react' +import StyledModal, { StyledModalProps } from './StyledModal' +import LoadingSpinner from './LoadingSpinner' +import api from '~/lib/api' +import { formatBytes } from '~/lib/util' +import classNames from '~/lib/classNames' +import { + EXTRACT_DEFAULT_MAX_ZOOM, + EXTRACT_MAX_ZOOM, + EXTRACT_MIN_ZOOM, +} from '../../constants/map_regions' +import type { + Country, + CountryCode, + CountryGroup, + MapExtractPreflight, +} from '../../types/maps' + +export type CountryPickerModalProps = Omit< + StyledModalProps, + | 'onConfirm' + | 'open' + | 'confirmText' + | 'cancelText' + | 'confirmVariant' + | 'children' + | 'title' + | 'large' +> & { + onDownloadStart?: () => void +} + +const CountryPickerModal: React.FC = ({ + onDownloadStart, + ...modalProps +}) => { + const [selected, setSelected] = useState>(new Set()) + const [search, setSearch] = useState('') + const [maxzoom, setMaxzoom] = useState(EXTRACT_DEFAULT_MAX_ZOOM) + const [preflight, setPreflight] = useState(null) + const [loading, setLoading] = useState(false) + const [downloading, setDownloading] = useState(false) + const [errorMessage, setErrorMessage] = useState(null) + const preflightRequestIdRef = useRef(0) + + const { data: countries = [], isLoading: countriesLoading } = useQuery({ + queryKey: ['maps-countries'], + queryFn: () => api.listCountries(), + staleTime: Infinity, + }) + + const { data: groups = [] } = useQuery({ + queryKey: ['maps-country-groups'], + queryFn: () => api.listCountryGroups(), + staleTime: Infinity, + }) + + const grouped = useMemo(() => { + const q = search.trim().toLowerCase() + const filtered = q + ? countries.filter( + (c) => c.name.toLowerCase().includes(q) || c.code.toLowerCase().includes(q) + ) + : countries + + const buckets: Record = {} + for (const country of filtered) { + if (!buckets[country.continent]) buckets[country.continent] = [] + buckets[country.continent].push(country) + } + return Object.entries(buckets).sort(([a], [b]) => a.localeCompare(b)) + }, [countries, search]) + + const selectedCountries = useMemo( + () => countries.filter((c) => selected.has(c.code)), + [countries, selected] + ) + + function toggleCountry(code: CountryCode) { + setSelected((prev) => { + const next = new Set(prev) + if (next.has(code)) next.delete(code) + else next.add(code) + return next + }) + } + + function toggleGroup(group: CountryGroup) { + setSelected((prev) => { + const next = new Set(prev) + const allIn = group.countries.every((c) => next.has(c)) + if (allIn) { + group.countries.forEach((c) => next.delete(c)) + } else { + group.countries.forEach((c) => next.add(c)) + } + return next + }) + } + + function clearAll() { + setSelected(new Set()) + } + + // Auto-refresh the preflight whenever selection or maxzoom changes. Debounced + // so rapid multi-select clicks collapse into a single CDN round-trip, and + // stale-safe via requestId so an earlier slow response can't clobber a later one. + useEffect(() => { + if (selected.size === 0) { + setPreflight(null) + setErrorMessage(null) + setLoading(false) + preflightRequestIdRef.current++ + return + } + + const requestId = ++preflightRequestIdRef.current + setLoading(true) + setErrorMessage(null) + const timer = setTimeout(async () => { + try { + const res = await api.extractMapPreflight({ + countries: [...selected], + maxzoom, + }) + if (requestId !== preflightRequestIdRef.current) return + if (!res) throw new Error('Preflight returned no data') + setPreflight(res) + } catch (err: any) { + if (requestId !== preflightRequestIdRef.current) return + console.error('Preflight failed:', err) + setErrorMessage(err?.message ?? 'Estimate failed') + } finally { + if (requestId === preflightRequestIdRef.current) setLoading(false) + } + }, 400) + + return () => clearTimeout(timer) + }, [selected, maxzoom]) + + async function startDownload() { + if (selected.size === 0) { + setErrorMessage('Pick at least one country before downloading.') + return + } + if (loading || !preflight) { + setErrorMessage('Still estimating size — hold on a moment.') + return + } + try { + setDownloading(true) + setErrorMessage(null) + await api.extractMapRegion({ + countries: [...selected], + maxzoom, + estimatedBytes: preflight?.bytes, + }) + onDownloadStart?.() + } catch (err: any) { + console.error('Extract dispatch failed:', err) + setErrorMessage(err?.message ?? 'Download failed') + } finally { + setDownloading(false) + } + } + + return ( + +

+
+
+ + setSearch(e.target.value)} + placeholder={`Search ${countries.length} countries...`} + className="w-full pl-9 pr-3 py-2 rounded-md border border-border-default bg-surface-primary text-text-primary text-sm focus:outline-none focus:ring-2 focus:ring-desert-green" + /> +
+ {selected.size > 0 && ( + + )} +
+ + {groups.length > 0 && ( +
+

+ Quick picks +

+
+ {groups.map((group) => { + const allIn = + group.countries.length > 0 && + group.countries.every((c) => selected.has(c)) + return ( + + ) + })} +
+
+ )} + +
+ {countriesLoading ? ( +
+ +
+ ) : grouped.length === 0 ? ( +

+ No countries match "{search}". +

+ ) : ( + grouped.map(([continent, list]) => ( +
+
+ {continent} +
+
    + {list.map((country) => { + const isSelected = selected.has(country.code) + return ( +
  • + +
    • + ) + })} +
+
+ )) + )} +
+ + {selectedCountries.length > 0 && ( +
+

+ {selectedCountries.length} selected +

+
+ {selectedCountries.map((country) => ( + + {country.name} + + + ))} +
+
+ )} + +
+ + setMaxzoom(parseInt(e.target.value, 10))} + className="w-full accent-desert-green" + disabled={loading || downloading} + /> +
+ z{EXTRACT_MIN_ZOOM} (world) + z{EXTRACT_MAX_ZOOM} (street) +
+

+ Lower zoom = smaller file, less detail. Zoom 15 shows individual streets; + zoom 10 shows city-level detail. +

+
+ +
+ 0} + /> +
+ +
+ + ) +} + +type PreflightStatusProps = { + errorMessage: string | null + loading: boolean + preflight: MapExtractPreflight | null + hasSelection: boolean +} + +function PreflightStatus({ errorMessage, loading, preflight, hasSelection }: PreflightStatusProps) { + if (errorMessage) { + return

{errorMessage}

+ } + if (loading) { + return

Estimating size…

+ } + if (preflight) { + return ( +

+ {preflight.tiles.toLocaleString()} tiles, ~{formatBytes(preflight.bytes, 1)}{' '} + (source build {preflight.source.date}) +

+ ) + } + if (!hasSelection) { + return

Pick at least one country to estimate size.

+ } + return

Estimating size…

+} + +export default CountryPickerModal diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 2449259..77e0515 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -4,6 +4,7 @@ import { ServiceSlim } from '../../types/services' import { FileEntry } from '../../types/files' import { CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus } from '../../types/system' import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads' +import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps' import { EmbedJobWithProgress } from '../../types/rag' import type { CategoryWithStatus, CollectionWithStatus, ContentUpdateCheckResult, ResourceUpdateInfo } from '../../types/collections' import { catchInternal } from './util' @@ -557,6 +558,46 @@ class API { })() } + async listCountries() { + return catchInternal(async () => { + const response = await this.client.get<{ countries: Country[] }>('/maps/countries') + return response.data.countries + })() + } + + async listCountryGroups() { + return catchInternal(async () => { + const response = await this.client.get<{ groups: CountryGroup[] }>('/maps/country-groups') + return response.data.groups + })() + } + + async extractMapPreflight(params: { countries: CountryCode[]; maxzoom?: number }) { + return catchInternal(async () => { + const response = await this.client.post( + '/maps/extract-preflight', + params + ) + return response.data + })() + } + + async extractMapRegion(params: { + countries: CountryCode[] + maxzoom?: number + label?: string + estimatedBytes?: number + }) { + return catchInternal(async () => { + const response = await this.client.post<{ + message: string + filename: string + jobId?: string + }>('/maps/extract', params) + return response.data + })() + } + async listCuratedMapCollections() { return catchInternal(async () => { const response = await this.client.get( diff --git a/admin/inertia/pages/settings/maps.tsx b/admin/inertia/pages/settings/maps.tsx index 47055fa..dcd945f 100644 --- a/admin/inertia/pages/settings/maps.tsx +++ b/admin/inertia/pages/settings/maps.tsx @@ -13,6 +13,7 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import useDownloads from '~/hooks/useDownloads' import StyledSectionHeader from '~/components/StyledSectionHeader' import CuratedCollectionCard from '~/components/CuratedCollectionCard' +import CountryPickerModal from '~/components/CountryPickerModal' import type { CollectionWithStatus } from '../../../types/collections' import ActiveDownloads from '~/components/ActiveDownloads' import Alert from '~/components/Alert' @@ -221,6 +222,23 @@ export default function MapsManager(props: { ) } + function openCountryPickerModal() { + openModal( + { + invalidateDownloads() + addNotification({ + type: 'success', + message: 'Download queued. Watch progress below.', + }) + closeAllModals() + }} + />, + 'country-picker-modal' + ) + } + async function openDownloadModal() { openModal( )} + +
Date: Sun, 3 May 2026 14:06:56 -0700 Subject: [PATCH 07/18] fix(UI): Country Picker UX polish + auto-refresh stored files (#817) Three UX issues from manual testing of #780 on NOMAD3. 1. Slider was unusable for multi-step zoom changes `setLoading(true)` fired immediately on every selection or maxzoom change, which disabled the slider until the request returned. Even with the 400ms debounce delaying the network call, the UI was locked the whole time. User couldn't drag through zoom levels to find the right one. Fix: bump debounce to 1500ms, move `setLoading(true)` inside the setTimeout so it only flips after the debounce expires. Slider stays interactive throughout the wait. Slider `disabled` now only ties to `downloading` (active extract dispatch), not `loading` (preflight in flight). The existing requestId stale-safe pattern handles concurrent changes. 2. Newly-downloaded maps didn't show in Stored Map Files until manual refresh `props.maps.regionFiles` is rendered server-side and passed through Inertia props; without a partial reload it stayed stale until the user navigated away and back. Fix: watch `useDownloads({ filetype: 'map' })` count via a ref. When the count drops (a download finished), trigger `router.reload({ only: ['maps'] })` to refresh just the maps prop. Existing pattern from elsewhere in the codebase. 3. Country picker didn't surface already-downloaded countries When a user re-opened "Choose Countries" after downloading UK, UK appeared unchecked with no indication it was already on disk. Fix: pass installed pmtiles filenames into the modal as a prop; parse with regex `^([a-z]{2})_[\w-]+_z\d+\.pmtiles$` to extract country codes from single-country extracts (matching MapService.buildRegionSlug's iso2 lowercase slug pattern). Render an "Installed" badge on those countries with a tooltip explaining they're re-selectable for redownload at a different zoom. Group / custom multi-country extracts don't reverse-map cleanly from filename and are skipped here. Could be a follow-up if useful. Files: admin/inertia/components/CountryPickerModal.tsx - SINGLE_COUNTRY_FILENAME_RE: iso2 + flexible date + zoom - installedFilenames prop with default [] - installedCountrySet derivation via useMemo - "Installed" badge rendering on country list rows - Debounce: 400ms -> 1500ms; setLoading inside setTimeout - Slider disabled: only on `downloading` admin/inertia/pages/settings/maps.tsx - import useEffect/useRef - destructure activeMapDownloads from useDownloads - useEffect on download count drop -> router.reload({ only: ['maps'] }) - pass installedFilenames to CountryPickerModal All three fixes tested end-to-end on NOMAD3. --- .../inertia/components/CountryPickerModal.tsx | 42 ++++++++++++++++--- admin/inertia/pages/settings/maps.tsx | 16 ++++++- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/admin/inertia/components/CountryPickerModal.tsx b/admin/inertia/components/CountryPickerModal.tsx index b490360..1b29334 100644 --- a/admin/inertia/components/CountryPickerModal.tsx +++ b/admin/inertia/components/CountryPickerModal.tsx @@ -30,10 +30,20 @@ export type CountryPickerModalProps = Omit< | 'large' > & { onDownloadStart?: () => void + /** Filenames of pmtiles already on disk; used to badge already-installed countries. */ + installedFilenames?: string[] } +// Single-country extracts use the slug `{iso2 lowercase}_{dateSlug}_z{maxzoom}.pmtiles`, +// matching MapService.buildRegionSlug (which lowercases the alpha-2 country code). +// dateSlug comes from the upstream pmtiles key with `.pmtiles` stripped — currently +// YYYYMMDD but we accept any digits/dashes. Group / custom filenames don't reverse-map +// to country codes, so we skip them here. +const SINGLE_COUNTRY_FILENAME_RE = /^([a-z]{2})_[\w-]+_z\d+\.pmtiles$/ + const CountryPickerModal: React.FC = ({ onDownloadStart, + installedFilenames = [], ...modalProps }) => { const [selected, setSelected] = useState>(new Set()) @@ -78,6 +88,15 @@ const CountryPickerModal: React.FC = ({ [countries, selected] ) + const installedCountrySet = useMemo(() => { + const set = new Set() + for (const filename of installedFilenames) { + const match = SINGLE_COUNTRY_FILENAME_RE.exec(filename) + if (match) set.add(match[1].toUpperCase() as CountryCode) + } + return set + }, [installedFilenames]) + function toggleCountry(code: CountryCode) { setSelected((prev) => { const next = new Set(prev) @@ -105,8 +124,10 @@ const CountryPickerModal: React.FC = ({ } // Auto-refresh the preflight whenever selection or maxzoom changes. Debounced - // so rapid multi-select clicks collapse into a single CDN round-trip, and - // stale-safe via requestId so an earlier slow response can't clobber a later one. + // so rapid multi-select clicks and slider drags collapse into a single CDN + // round-trip. Loading state only flips after the debounce expires so the UI + // stays interactive during the wait. Stale-safe via requestId so an earlier + // slow response can't clobber a later one. useEffect(() => { if (selected.size === 0) { setPreflight(null) @@ -116,10 +137,10 @@ const CountryPickerModal: React.FC = ({ return } - const requestId = ++preflightRequestIdRef.current - setLoading(true) setErrorMessage(null) const timer = setTimeout(async () => { + const requestId = ++preflightRequestIdRef.current + setLoading(true) try { const res = await api.extractMapPreflight({ countries: [...selected], @@ -135,7 +156,7 @@ const CountryPickerModal: React.FC = ({ } finally { if (requestId === preflightRequestIdRef.current) setLoading(false) } - }, 400) + }, 1500) return () => clearTimeout(timer) }, [selected, maxzoom]) @@ -253,6 +274,7 @@ const CountryPickerModal: React.FC = ({
    {list.map((country) => { const isSelected = selected.has(country.code) + const isInstalled = installedCountrySet.has(country.code) return (
  • - -
- + { + setIsDraggingMap(true) + hideCoordinates() + }} + onMouseUp={() => { + setIsDraggingMap(false) + }} + onDragStart={() => { + setIsDraggingMap(true) + hideCoordinates() + }} + onDragEnd={() => { + setIsDraggingMap(false) + hideCoordinates() + }} + onClick={handleMapClick} + onMouseMove={handleMouseMove} + onMouseLeave={hideCoordinates} + > + + + - {/* Existing markers */} - {markers.map((marker) => ( - { - e.originalEvent.stopPropagation() - setSelectedMarkerId(marker.id === selectedMarkerId ? null : marker.id) - setPlacingMarker(null) - }} - > - c.id === marker.color)?.hex} - active={marker.id === selectedMarkerId} + {showCoordinates && cursorLngLat && ( + - - ))} + )} - {/* Popup for selected marker */} - {selectedMarker && ( - setSelectedMarkerId(null)} - closeOnClick={false} - > -
{selectedMarker.name}
- - )} + - {/* Popup for placing a new marker */} - {placingMarker && ( - setPlacingMarker(null)} - closeOnClick={false} - > -
- setMarkerName(e.target.value)} - onKeyDown={(e) => { - if (e.key === 'Enter') handleSaveMarker() - if (e.key === 'Escape') setPlacingMarker(null) - }} - className="block w-full rounded border border-gray-300 px-2 py-1 text-sm placeholder:text-gray-400 focus:outline-none focus:border-gray-500" + {markers.map((marker) => ( + { + e.originalEvent.stopPropagation() + setSelectedMarkerId(marker.id === selectedMarkerId ? null : marker.id) + setPlacingMarker(null) + }} + > + c.id === marker.color)?.hex} + active={marker.id === selectedMarkerId} /> -
- {PIN_COLORS.map((c) => ( - - ))} -
-
- - -
-
- - )} - + + ))} - {/* Marker panel overlay */} - + {selectedMarker && ( + setSelectedMarkerId(null)} + closeOnClick={false} + > +
{selectedMarker.name}
+ + )} + + {placingMarker && ( + setPlacingMarker(null)} + closeOnClick={false} + > +
+ setMarkerName(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') handleSaveMarker() + if (e.key === 'Escape') setPlacingMarker(null) + }} + className="block w-full rounded border border-gray-300 px-2 py-1 text-sm placeholder:text-gray-400 focus:outline-none focus:border-gray-500" + /> + +
+ {PIN_COLORS.map((c) => ( + + ))} +
+ +
+ + + +
+
+ + )} + + + +
+ +
) } diff --git a/admin/inertia/components/maps/ScaleUnitToggle.tsx b/admin/inertia/components/maps/ScaleUnitToggle.tsx new file mode 100644 index 0000000..ba0bceb --- /dev/null +++ b/admin/inertia/components/maps/ScaleUnitToggle.tsx @@ -0,0 +1,46 @@ +type ScaleUnit = 'imperial' | 'metric' + +type ScaleUnitToggleProps = { + scaleUnit: ScaleUnit + onChange: (unit: ScaleUnit) => void + onMouseEnter?: () => void +} + +export default function ScaleUnitToggle({ + scaleUnit, + onChange, + onMouseEnter, +}: ScaleUnitToggleProps) { + return ( +
+
+ + + +
+
+ ) +} diff --git a/admin/inertia/pages/maps.tsx b/admin/inertia/pages/maps.tsx index 9fe20e0..a1d8df1 100644 --- a/admin/inertia/pages/maps.tsx +++ b/admin/inertia/pages/maps.tsx @@ -1,38 +1,66 @@ -import MapsLayout from '~/layouts/MapsLayout' +import { useState } from 'react' import { Head, Link, router } from '@inertiajs/react' +import { IconArrowLeft } from '@tabler/icons-react' + +import MapsLayout from '~/layouts/MapsLayout' import MapComponent from '~/components/maps/MapComponent' import StyledButton from '~/components/StyledButton' -import { IconArrowLeft } from '@tabler/icons-react' -import { FileEntry } from '../../types/files' import Alert from '~/components/Alert' +import { FileEntry } from '../../types/files' + export default function Maps(props: { maps: { baseAssetsExist: boolean; regionFiles: FileEntry[] } }) { + const [isHoveringUI, setIsHoveringUI] = useState(false) + const [showMapCoordinates, setShowMapCoordinates] = useState(true) + const alertMessage = !props.maps.baseAssetsExist ? 'The base map assets have not been installed. Please download them first to enable map functionality.' : props.maps.regionFiles.length === 0 - ? 'No map regions have been downloaded yet. Please download some regions to enable map functionality.' - : null + ? 'No map regions have been downloaded yet. Please download some regions to enable map functionality.' + : null return ( +
- {/* Nav and alerts are overlayed */} -
+ {/* Navbar */} +
setIsHoveringUI(true)} + onMouseLeave={() => setIsHoveringUI(false)} + >

Back to Home

- - - Manage Map Regions - - + +
+ + + + + Manage Map Regions + + +
+ + {/* Alert */} {alertMessage && ( -
+
setIsHoveringUI(true)} + onMouseLeave={() => setIsHoveringUI(false)} + >
)} + + {/* Map */}
- +
From 8ef2c69f56a155830a4c8fb46c8fc0ed78861023 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Wed, 29 Apr 2026 15:52:05 -0700 Subject: [PATCH 09/18] docs: link to new WSL2 install guide from README and FAQ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /install/wsl2 community-supported install guide is now live on projectnomad.us. Update the README install section and FAQ to point Windows users at it instead of deflecting WSL2 questions to "see the Debian-only answer." Doesn't change the official-support stance — bare-metal Debian-based Linux remains the supported configuration. Just removes the dead-end deflection so Windows users have a real path forward. Co-Authored-By: Claude Opus 4.7 (1M context) --- FAQ.md | 6 ++++-- README.md | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/FAQ.md b/FAQ.md index fee06f8..1823c7b 100644 --- a/FAQ.md +++ b/FAQ.md @@ -20,7 +20,9 @@ Long answer: Custom storage paths, mount points, and external drives (like iSCSI ## Can I run NOMAD on MAC, WSL2, or a non-Debian-based Distro? -See [Why does NOMAD require a Debian-based OS?](#why-does-nomad-require-a-debian-based-os) +**WSL2 on Windows** is community-supported via the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) — covers two install paths (native Docker and Docker Desktop) with all known gotchas documented and empirical performance numbers comparing WSL2 to bare-metal. + +**macOS and other non-Debian Linux distros** aren't officially supported. See [Why does NOMAD require a Debian-based OS?](#why-does-nomad-require-a-debian-based-os) for details. ## Why does NOMAD require a Debian-based OS? @@ -28,7 +30,7 @@ Project N.O.M.A.D. is currently designed to run on Debian-based Linux distributi Support for other operating systems will come in the future, but because our development resources are limited as a free and open-source project, we needed to prioritize our efforts and focus on a narrower set of supported platforms for the initial release. We chose Debian-based Linux as our starting point because it's widely used, easy to spin up, and provides a stable environment for running Docker containers. -Community members have provided guides for running N.O.M.A.D. on other platforms (e.g. WSL2, Mac, etc.) in our Discord community and [Github Discussions](https://github.com/Crosstalk-Solutions/project-nomad/discussions), so if you're interested in running N.O.M.A.D. on a non-Debian-based system, we recommend checking there for any available resources or guides. However, keep in mind that if you choose to run N.O.M.A.D. on a non-Debian-based system, you may encounter issues that we won't be able to provide support for, and you may need to have a higher level of technical expertise to troubleshoot and resolve any problems that arise. +For Windows users, the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) provides a community-supported path. Community members have also published guides for other platforms (e.g. macOS) in our Discord community and [Github Discussions](https://github.com/Crosstalk-Solutions/project-nomad/discussions), so if you're interested in running N.O.M.A.D. on a non-Debian-based system, we recommend checking there for any available resources or guides. However, keep in mind that if you choose to run N.O.M.A.D. on a non-Debian-based system, you may encounter issues that we won't be able to provide support for, and you may need to have a higher level of technical expertise to troubleshoot and resolve any problems that arise. ## Can I run NOMAD on a Raspberry Pi or other ARM-based device? Project N.O.M.A.D. is currently designed to run on x86-64 architecture, and we have not yet tested or optimized it for ARM-based devices like the Raspberry Pi (and have not published any official images for ARM architecture). diff --git a/README.md b/README.md index 0edd05a..a7e1100 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ sudo bash install_nomad.sh Project N.O.M.A.D. is now installed on your device! Open a browser and navigate to `http://localhost:8080` (or `http://DEVICE_IP:8080`) to start exploring! -For a complete step-by-step walkthrough (including Ubuntu installation), see the [Installation Guide](https://www.projectnomad.us/install). +For a complete step-by-step walkthrough (including Ubuntu installation), see the [Installation Guide](https://www.projectnomad.us/install). For Windows users, see the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) — community-supported path covering native Docker and Docker Desktop install routes. ### Advanced Installation For more control over the installation process, copy and paste the [Docker Compose template](https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/install/management_compose.yaml) into a `docker-compose.yml` file and customize it to your liking (be sure to replace any placeholders with your actual values). Then, run `docker compose up -d` to start the Command Center and its dependencies. Note: this method is recommended for advanced users only, as it requires familiarity with Docker and manual configuration before starting. From d81b66bb145cf63e3bec2cfb21adbe3ed0658216 Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Mon, 4 May 2026 17:45:18 +0000 Subject: [PATCH 10/18] chore(deps): pin all deps to exact versions --- admin/package.json | 170 ++++++++++++++++++++++----------------------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/admin/package.json b/admin/package.json index d98e7e0..b273fd0 100644 --- a/admin/package.json +++ b/admin/package.json @@ -38,94 +38,94 @@ "#jobs/*": "./app/jobs/*.js" }, "devDependencies": { - "@adonisjs/assembler": "^7.8.2", - "@adonisjs/eslint-config": "^2.0.0", - "@adonisjs/prettier-config": "^1.4.4", - "@adonisjs/tsconfig": "^1.4.0", - "@japa/assert": "^4.0.1", - "@japa/plugin-adonisjs": "^4.0.0", - "@japa/runner": "^4.2.0", + "@adonisjs/assembler": "7.8.2", + "@adonisjs/eslint-config": "2.1.2", + "@adonisjs/prettier-config": "1.4.5", + "@adonisjs/tsconfig": "1.4.1", + "@japa/assert": "4.2.0", + "@japa/plugin-adonisjs": "4.0.0", + "@japa/runner": "4.5.0", "@swc/core": "1.11.24", - "@tanstack/eslint-plugin-query": "^5.81.2", - "@types/compression": "^1.8.1", - "@types/dockerode": "^4.0.1", - "@types/luxon": "^3.6.2", - "@types/node": "^22.15.18", - "@types/react": "^19.1.8", - "@types/react-dom": "^19.1.6", - "@types/stopword": "^2.0.3", - "eslint": "^9.26.0", - "hot-hook": "^0.4.0", - "prettier": "^3.5.3", - "ts-node-maintained": "^10.9.5", - "typescript": "~5.8.3", - "vite": "^6.4.2" + "@tanstack/eslint-plugin-query": "5.91.4", + "@types/compression": "1.8.1", + "@types/dockerode": "4.0.1", + "@types/luxon": "3.7.1", + "@types/node": "22.19.7", + "@types/react": "19.2.10", + "@types/react-dom": "19.2.3", + "@types/stopword": "2.0.3", + "eslint": "9.39.2", + "hot-hook": "0.4.0", + "prettier": "3.8.1", + "ts-node-maintained": "10.9.6", + "typescript": "5.8.3", + "vite": "6.4.2" }, "dependencies": { - "@adonisjs/auth": "^9.4.0", - "@adonisjs/core": "^6.18.0", - "@adonisjs/cors": "^2.2.1", - "@adonisjs/inertia": "^3.1.1", - "@adonisjs/lucid": "^21.8.2", - "@adonisjs/session": "^7.5.1", - "@adonisjs/shield": "^8.2.0", - "@adonisjs/static": "^1.1.1", - "@adonisjs/transmit": "^2.0.2", - "@adonisjs/transmit-client": "^1.0.0", - "@adonisjs/vite": "^4.0.0", - "@chonkiejs/core": "^0.0.7", - "@headlessui/react": "^2.2.4", - "@inertiajs/react": "^2.0.13", - "@markdoc/markdoc": "^0.5.2", - "@openzim/libzim": "^4.0.0", - "@protomaps/basemaps": "^5.7.0", - "@qdrant/js-client-rest": "^1.16.2", - "@tabler/icons-react": "^3.34.0", - "@tailwindcss/vite": "^4.1.10", - "@tanstack/react-query": "^5.81.5", - "@tanstack/react-query-devtools": "^5.83.0", - "@tanstack/react-virtual": "^3.13.12", - "@uppy/core": "^5.2.0", - "@uppy/dashboard": "^5.1.0", - "@uppy/react": "^5.1.1", - "@vinejs/vine": "^3.0.1", - "@vitejs/plugin-react": "^4.6.0", - "autoprefixer": "^10.4.21", - "axios": "^1.15.0", - "better-sqlite3": "^12.1.1", - "bullmq": "^5.65.1", - "cheerio": "^1.2.0", - "compression": "^1.8.1", - "dockerode": "^4.0.7", - "edge.js": "^6.2.1", - "fast-xml-parser": "^5.5.7", - "fuse.js": "^7.1.0", - "jszip": "^3.10.1", - "luxon": "^3.6.1", - "maplibre-gl": "^4.7.1", - "mysql2": "^3.14.1", - "ollama": "^0.6.3", - "openai": "^6.27.0", - "pdf-parse": "^2.4.5", - "pdf2pic": "^3.2.0", - "pino-pretty": "^13.0.0", - "pmtiles": "^4.4.0", - "postcss": "^8.5.6", - "react": "^19.1.0", - "react-adonis-transmit": "^1.0.1", - "react-dom": "^19.1.0", - "react-map-gl": "^8.1.0", - "react-markdown": "^10.1.0", - "reflect-metadata": "^0.2.2", - "remark-gfm": "^4.0.1", - "sharp": "^0.34.5", - "stopword": "^3.1.5", - "systeminformation": "^5.31.0", - "tailwindcss": "^4.2.1", - "tar": "^7.5.11", - "tesseract.js": "^7.0.0", - "url-join": "^5.0.0", - "yaml": "^2.8.3" + "@adonisjs/auth": "9.6.0", + "@adonisjs/core": "6.19.3", + "@adonisjs/cors": "2.2.1", + "@adonisjs/inertia": "3.1.1", + "@adonisjs/lucid": "21.8.2", + "@adonisjs/session": "7.7.1", + "@adonisjs/shield": "8.2.0", + "@adonisjs/static": "1.1.1", + "@adonisjs/transmit": "2.0.2", + "@adonisjs/transmit-client": "1.1.0", + "@adonisjs/vite": "4.0.0", + "@chonkiejs/core": "0.0.7", + "@headlessui/react": "2.2.9", + "@inertiajs/react": "2.3.13", + "@markdoc/markdoc": "0.5.4", + "@openzim/libzim": "4.0.0", + "@protomaps/basemaps": "5.7.0", + "@qdrant/js-client-rest": "1.16.2", + "@tabler/icons-react": "3.36.1", + "@tailwindcss/vite": "4.1.18", + "@tanstack/react-query": "5.90.20", + "@tanstack/react-query-devtools": "5.91.3", + "@tanstack/react-virtual": "3.13.18", + "@uppy/core": "5.2.0", + "@uppy/dashboard": "5.1.0", + "@uppy/react": "5.1.1", + "@vinejs/vine": "3.0.1", + "@vitejs/plugin-react": "4.7.0", + "autoprefixer": "10.4.24", + "axios": "1.15.0", + "better-sqlite3": "12.6.2", + "bullmq": "5.67.2", + "cheerio": "1.2.0", + "compression": "1.8.1", + "dockerode": "4.0.9", + "edge.js": "6.4.0", + "fast-xml-parser": "5.5.9", + "fuse.js": "7.1.0", + "jszip": "3.10.1", + "luxon": "3.7.2", + "maplibre-gl": "4.7.1", + "mysql2": "3.16.2", + "ollama": "0.6.3", + "openai": "6.27.0", + "pdf-parse": "2.4.5", + "pdf2pic": "3.2.0", + "pino-pretty": "13.1.3", + "pmtiles": "4.4.0", + "postcss": "8.5.6", + "react": "19.2.4", + "react-adonis-transmit": "1.0.1", + "react-dom": "19.2.4", + "react-map-gl": "8.1.0", + "react-markdown": "10.1.0", + "reflect-metadata": "0.2.2", + "remark-gfm": "4.0.1", + "sharp": "0.34.5", + "stopword": "3.1.5", + "systeminformation": "5.31.0", + "tailwindcss": "4.2.2", + "tar": "7.5.11", + "tesseract.js": "7.0.0", + "url-join": "5.0.0", + "yaml": "2.8.3" }, "hotHook": { "boundaries": [ From d66eaa3d423ab049b1c382c3753375f3e8bdad13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Mar 2026 21:51:08 +0000 Subject: [PATCH 11/18] build(deps): bump picomatch in /admin Bumps and [picomatch](https://github.com/micromatch/picomatch). These dependencies needed to be updated together. Updates `picomatch` from 4.0.3 to 4.0.4 - [Release notes](https://github.com/micromatch/picomatch/releases) - [Changelog](https://github.com/micromatch/picomatch/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/picomatch/compare/4.0.3...4.0.4) Updates `picomatch` from 2.3.1 to 2.3.2 - [Release notes](https://github.com/micromatch/picomatch/releases) - [Changelog](https://github.com/micromatch/picomatch/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/picomatch/compare/4.0.3...4.0.4) --- updated-dependencies: - dependency-name: picomatch dependency-version: 4.0.4 dependency-type: indirect - dependency-name: picomatch dependency-version: 2.3.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- admin/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/admin/package-lock.json b/admin/package-lock.json index ae7ef36..29f8dd3 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -12174,9 +12174,9 @@ } }, "node_modules/micromatch/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "license": "MIT", "engines": { "node": ">=8.6" @@ -13351,9 +13351,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "license": "MIT", "engines": { "node": ">=12" From a7dbee55c4286e13b3915a88adf74bc259adff82 Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Mon, 4 May 2026 11:30:59 -0700 Subject: [PATCH 12/18] feat(Content): custom ZIM library sources with pre-seeded mirrors (#593) * feat(content): add custom ZIM library sources with pre-seeded mirrors Users reported slow download speeds from the default Kiwix CDN. This adds the ability to browse and download ZIM files from alternative Kiwix mirrors or self-hosted repositories, all through the GUI. - Add "Custom Libraries" button next to "Browse the Kiwix Library" - Source dropdown to switch between Default (Kiwix) and custom libraries - Browsable directory structure with breadcrumb navigation - 5 pre-seeded official Kiwix mirrors (US, DE, DK, UK, Global CDN) - Built-in mirrors protected from deletion - Downloads use existing pipeline (progress, cancel, Kiwix restart) - Source selection persists across page loads via localStorage - Scrollable directory browser (600px max) with sticky header - SSRF protection on all custom library URLs Closes #576 Co-Authored-By: Claude Opus 4.6 (1M context) * fix(content): recognize Wikipedia downloads from mirror sources When Wikipedia is downloaded via a custom mirror instead of the default Kiwix server, the completion callback now matches by filename instead of exact URL. This ensures the Wikipedia selector correctly shows "Installed" status and triggers old-version cleanup regardless of which mirror was used. Also handles the case where no Wikipedia selection exists yet (file downloaded before visiting the selector), creating the record automatically. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(ZIM): use cheerio for custom mirror directory parsing * fix(ZIM): use URL constructor for more robust joining --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: Jake Turner --- admin/app/controllers/zim_controller.ts | 49 +- admin/app/models/custom_library_source.ts | 24 + admin/app/services/zim_service.ts | 180 +++++- admin/app/validators/zim.ts | 27 + ...001_create_custom_library_sources_table.ts | 42 ++ admin/inertia/lib/api.ts | 36 ++ .../pages/settings/zim/remote-explorer.tsx | 545 +++++++++++++++--- admin/start/routes.ts | 6 + 8 files changed, 814 insertions(+), 95 deletions(-) create mode 100644 admin/app/models/custom_library_source.ts create mode 100644 admin/database/migrations/1775100000001_create_custom_library_sources_table.ts diff --git a/admin/app/controllers/zim_controller.ts b/admin/app/controllers/zim_controller.ts index 96adf63..006e59b 100644 --- a/admin/app/controllers/zim_controller.ts +++ b/admin/app/controllers/zim_controller.ts @@ -6,7 +6,7 @@ import { remoteDownloadWithMetadataValidator, selectWikipediaValidator, } from '#validators/common' -import { listRemoteZimValidator } from '#validators/zim' +import { addCustomLibraryValidator, browseLibraryValidator, idParamValidator, listRemoteZimValidator } from '#validators/zim' import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' @@ -85,4 +85,51 @@ export default class ZimController { const payload = await request.validateUsing(selectWikipediaValidator) return this.zimService.selectWikipedia(payload.optionId) } + + // Custom library endpoints + + async listCustomLibraries({}: HttpContext) { + return this.zimService.listCustomLibraries() + } + + async addCustomLibrary({ request, response }: HttpContext) { + const payload = await request.validateUsing(addCustomLibraryValidator) + assertNotPrivateUrl(payload.base_url) + try { + const source = await this.zimService.addCustomLibrary(payload.name, payload.base_url) + return { message: 'Custom library added', library: source } + } catch (error) { + if (error.message === 'Maximum of 10 custom libraries allowed') { + return response.status(400).send({ message: error.message }) + } + throw error + } + } + + async removeCustomLibrary({ request, response }: HttpContext) { + const payload = await request.validateUsing(idParamValidator) + try { + await this.zimService.removeCustomLibrary(payload.params.id) + return { message: 'Custom library removed' } + } catch (error) { + if (error.message === 'Custom library not found') { + return response.status(404).send({ message: error.message }) + } + throw error + } + } + + async browseLibrary({ request, response }: HttpContext) { + const payload = await request.validateUsing(browseLibraryValidator) + try { + return await this.zimService.browseLibraryUrl(payload.url) + } catch (error) { + if (error.message?.includes('loopback or link-local')) { + return response.status(400).send({ message: error.message }) + } + return response.status(502).send({ + message: 'Could not fetch directory listing from the provided URL', + }) + } + } } diff --git a/admin/app/models/custom_library_source.ts b/admin/app/models/custom_library_source.ts new file mode 100644 index 0000000..478d9a0 --- /dev/null +++ b/admin/app/models/custom_library_source.ts @@ -0,0 +1,24 @@ +import { DateTime } from 'luxon' +import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' + +export default class CustomLibrarySource extends BaseModel { + static namingStrategy = new SnakeCaseNamingStrategy() + + @column({ isPrimary: true }) + declare id: number + + @column() + declare name: string + + @column() + declare base_url: string + + @column() + declare is_default: boolean + + @column.dateTime({ autoCreate: true }) + declare created_at: DateTime + + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updated_at: DateTime +} diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index db6b5b7..1cc9e97 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -4,6 +4,7 @@ import { RemoteZimFileEntry, } from '../../types/zim.js' import axios from 'axios' +import * as cheerio from 'cheerio' import { XMLParser } from 'fast-xml-parser' import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from '../../util/zim.js' import logger from '@adonisjs/core/services/logger' @@ -27,6 +28,8 @@ import { SERVICE_NAMES } from '../../constants/service_names.js' import { CollectionManifestService } from './collection_manifest_service.js' import { KiwixLibraryService } from './kiwix_library_service.js' import type { CategoryWithStatus } from '../../types/collections.js' +import CustomLibrarySource from '#models/custom_library_source' +import { assertNotPrivateUrl } from '#validators/common' const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream'] const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json' @@ -587,25 +590,47 @@ export class ZimService { } async onWikipediaDownloadComplete(url: string, success: boolean): Promise { + const filename = url.split('/').pop() || '' const selection = await this.getWikipediaSelection() - if (!selection || selection.url !== url) { - logger.warn(`[ZimService] Wikipedia download complete callback for unknown URL: ${url}`) - return + // Determine which Wikipedia option this file belongs to by matching filename + let matchedOptionId: string | null = null + try { + const options = await this.getWikipediaOptions() + for (const opt of options) { + if (opt.url && opt.url.split('/').pop() === filename) { + matchedOptionId = opt.id + break + } + } + } catch { + // If we can't fetch options, try to continue with existing selection } if (success) { - // Update status to installed - selection.status = 'installed' - await selection.save() + // Update or create the selection record + // Match by filename (not URL) so mirror downloads are recognized + if (selection) { + selection.option_id = matchedOptionId || selection.option_id + selection.url = url + selection.filename = filename + selection.status = 'installed' + await selection.save() + } else { + await WikipediaSelection.create({ + option_id: matchedOptionId || 'unknown', + url: url, + filename: filename, + status: 'installed', + }) + } - logger.info(`[ZimService] Wikipedia download completed successfully: ${selection.filename}`) + logger.info(`[ZimService] Wikipedia download completed successfully: ${filename}`) - // Delete the old Wikipedia file if it exists and is different - // We need to find what was previously installed + // Delete old Wikipedia files (keep only the newly installed one) const existingFiles = await this.list() const wikipediaFiles = existingFiles.files.filter((f) => - f.name.startsWith('wikipedia_en_') && f.name !== selection.filename + f.name.startsWith('wikipedia_en_') && f.name !== filename ) for (const oldFile of wikipediaFiles) { @@ -617,10 +642,137 @@ export class ZimService { } } } else { - // Download failed - keep the selection record but mark as failed - selection.status = 'failed' - await selection.save() - logger.error(`[ZimService] Wikipedia download failed for: ${selection.filename}`) + // Download failed - update selection if it matches this file + if (selection && (!selection.filename || selection.filename === filename)) { + selection.status = 'failed' + await selection.save() + logger.error(`[ZimService] Wikipedia download failed for: ${filename}`) + } else { + logger.error(`[ZimService] Wikipedia download failed for: ${filename} (no matching selection)`) + } } } + + // Custom library source management + + async listCustomLibraries(): Promise { + return CustomLibrarySource.all() + } + + async addCustomLibrary(name: string, baseUrl: string): Promise { + const count = await CustomLibrarySource.query().count('* as total') + const total = Number(count[0].$extras.total) + if (total >= 10) { + throw new Error('Maximum of 10 custom libraries allowed') + } + + // Ensure URL ends with / + const normalizedUrl = baseUrl.endsWith('/') ? baseUrl : baseUrl + '/' + + return CustomLibrarySource.create({ + name, + base_url: normalizedUrl, + }) + } + + async removeCustomLibrary(id: number): Promise { + const source = await CustomLibrarySource.find(id) + if (!source) { + throw new Error('Custom library not found') + } + if (source.is_default) { + throw new Error('Cannot remove a built-in mirror') + } + await source.delete() + } + + async browseLibraryUrl(url: string): Promise<{ + directories: { name: string; url: string }[] + files: { name: string; url: string; size_bytes: number | null }[] + }> { + assertNotPrivateUrl(url) + + const normalizedUrl = url.endsWith('/') ? url : url + '/' + + const res = await axios.get(normalizedUrl, { + responseType: 'text', + timeout: 15000, + headers: { + 'Accept': 'text/html', + }, + }) + + const html: string = res.data + const directories: { name: string; url: string }[] = [] + const files: { name: string; url: string; size_bytes: number | null }[] = [] + + const $ = cheerio.load(html) + + $('a').each((_, el) => { + const href = el.attribs?.href + if (!href || href === '../' || href === './' || href === '/' || href.startsWith('?') || href.startsWith('#')) { + return + } + if (href.startsWith('/') || href.startsWith('http://') || href.startsWith('https://')) { + return + } + + if (href.endsWith('/')) { + const dirName = decodeURIComponent(href.replace(/\/$/, '')) + directories.push({ + name: dirName, + url: new URL(href, normalizedUrl).toString(), + }) + return + } + + if (href.endsWith('.zim')) { + const fileName = decodeURIComponent(href) + + // Apache/Nginx autoindex put the date + size in the text node directly + // following within a 
. Walk forward across text siblings until
+        // we find a parseable size token.
+        let trailingText = ''
+        let sibling = el.next
+        while (sibling && sibling.type === 'text') {
+          trailingText += sibling.data
+          if (/\n/.test(sibling.data)) break
+          sibling = sibling.next
+        }
+
+        files.push({
+          name: fileName,
+          url: new URL(href, normalizedUrl).toString(),
+          size_bytes: this._parseListingSize(trailingText),
+        })
+      }
+    })
+
+    directories.sort((a, b) => a.name.localeCompare(b.name))
+    files.sort((a, b) => a.name.localeCompare(b.name))
+
+    return { directories, files }
+  }
+
+  /**
+   * Parse a directory-listing size token out of the text that follows an anchor.
+   * Apache renders e.g. `   2024-01-15 10:30  5.1G`; Nginx renders raw bytes.
+   * Returns bytes or null if no size token is found.
+   */
+  private _parseListingSize(text: string): number | null {
+    // Skip the date/time columns; grab the last numeric token (with optional suffix)
+    // before a newline. Matches `5.1G`, `5368709120`, `1.2T`, etc.
+    const sizeMatch = /([\d.]+\s*[KMGT]?B?|\d+)\s*$/i.exec(text.split('\n')[0].trim())
+    if (!sizeMatch) return null
+
+    const sizeStr = sizeMatch[1].replace(/\s|B$/gi, '')
+    const num = parseFloat(sizeStr)
+    if (isNaN(num)) return null
+
+    if (/^\d+$/.test(sizeStr)) return num
+
+    const suffix = sizeStr.slice(-1).toUpperCase()
+    const multipliers: Record = { K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 }
+    return multipliers[suffix] ? Math.round(num * multipliers[suffix]) : null
+  }
 }
diff --git a/admin/app/validators/zim.ts b/admin/app/validators/zim.ts
index 2c18271..5463e52 100644
--- a/admin/app/validators/zim.ts
+++ b/admin/app/validators/zim.ts
@@ -7,3 +7,30 @@ export const listRemoteZimValidator = vine.compile(
     query: vine.string().optional(),
   })
 )
+
+export const addCustomLibraryValidator = vine.compile(
+  vine.object({
+    name: vine.string().trim().minLength(1).maxLength(100),
+    base_url: vine
+      .string()
+      .url({ require_tld: false })
+      .trim(),
+  })
+)
+
+export const browseLibraryValidator = vine.compile(
+  vine.object({
+    url: vine
+      .string()
+      .url({ require_tld: false })
+      .trim(),
+  })
+)
+
+export const idParamValidator = vine.compile(
+  vine.object({
+    params: vine.object({
+      id: vine.number(),
+    }),
+  })
+)
diff --git a/admin/database/migrations/1775100000001_create_custom_library_sources_table.ts b/admin/database/migrations/1775100000001_create_custom_library_sources_table.ts
new file mode 100644
index 0000000..baa3c77
--- /dev/null
+++ b/admin/database/migrations/1775100000001_create_custom_library_sources_table.ts
@@ -0,0 +1,42 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+export default class extends BaseSchema {
+  protected tableName = 'custom_library_sources'
+
+  async up() {
+    this.schema.createTable(this.tableName, (table) => {
+      table.increments('id').primary()
+      table.string('name', 100).notNullable()
+      table.string('base_url', 2048).notNullable()
+      table.boolean('is_default').notNullable().defaultTo(false)
+      table.timestamp('created_at').notNullable()
+      table.timestamp('updated_at').notNullable()
+    })
+
+    // Seed default Kiwix mirrors
+    const now = new Date().toISOString().slice(0, 19).replace('T', ' ')
+    const defaults = [
+      { name: 'Debian CDN (Global)', base_url: 'https://cdimage.debian.org/mirror/kiwix.org/zim/' },
+      { name: 'Your.org (US)', base_url: 'https://ftpmirror.your.org/pub/kiwix/zim/' },
+      { name: 'FAU Erlangen (DE)', base_url: 'https://ftp.fau.de/kiwix/zim/' },
+      { name: 'Dotsrc (DK)', base_url: 'https://mirrors.dotsrc.org/kiwix/zim/' },
+      { name: 'MirrorService (UK)', base_url: 'https://www.mirrorservice.org/sites/download.kiwix.org/zim/' },
+    ]
+
+    for (const d of defaults) {
+      await this.defer(async (db) => {
+        await db.table(this.tableName).insert({
+          name: d.name,
+          base_url: d.base_url,
+          is_default: true,
+          created_at: now,
+          updated_at: now,
+        })
+      })
+    }
+  }
+
+  async down() {
+    this.schema.dropTable(this.tableName)
+  }
+}
diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts
index 77e0515..3e015d8 100644
--- a/admin/inertia/lib/api.ts
+++ b/admin/inertia/lib/api.ts
@@ -681,6 +681,42 @@ class API {
     })()
   }
 
+  async listCustomLibraries() {
+    return catchInternal(async () => {
+      const response = await this.client.get<{ id: number; name: string; base_url: string; is_default: boolean }[]>(
+        '/zim/custom-libraries'
+      )
+      return response.data
+    })()
+  }
+
+  async addCustomLibrary(name: string, base_url: string) {
+    return catchInternal(async () => {
+      const response = await this.client.post<{
+        message: string
+        library: { id: number; name: string; base_url: string }
+      }>('/zim/custom-libraries', { name, base_url })
+      return response.data
+    })()
+  }
+
+  async removeCustomLibrary(id: number) {
+    return catchInternal(async () => {
+      const response = await this.client.delete<{ message: string }>(`/zim/custom-libraries/${id}`)
+      return response.data
+    })()
+  }
+
+  async browseLibrary(url: string) {
+    return catchInternal(async () => {
+      const response = await this.client.get<{
+        directories: { name: string; url: string }[]
+        files: { name: string; url: string; size_bytes: number | null }[]
+      }>('/zim/browse-library', { params: { url } })
+      return response.data
+    })()
+  }
+
   async deleteZimFile(filename: string) {
     return catchInternal(async () => {
       const response = await this.client.delete<{ message: string }>(`/zim/${filename}`)
diff --git a/admin/inertia/pages/settings/zim/remote-explorer.tsx b/admin/inertia/pages/settings/zim/remote-explorer.tsx
index 9d1ca3e..7fcfed1 100644
--- a/admin/inertia/pages/settings/zim/remote-explorer.tsx
+++ b/admin/inertia/pages/settings/zim/remote-explorer.tsx
@@ -21,7 +21,16 @@ import useInternetStatus from '~/hooks/useInternetStatus'
 import Alert from '~/components/Alert'
 import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
 import Input from '~/components/inputs/Input'
-import { IconSearch, IconBooks } from '@tabler/icons-react'
+import {
+  IconSearch,
+  IconBooks,
+  IconFolder,
+  IconFileDownload,
+  IconChevronRight,
+  IconPlus,
+  IconTrash,
+  IconLibrary,
+} from '@tabler/icons-react'
 import useDebounce from '~/hooks/useDebounce'
 import CategoryCard from '~/components/CategoryCard'
 import TierSelectionModal from '~/components/TierSelectionModal'
@@ -34,6 +43,13 @@ import { SERVICE_NAMES } from '../../../../constants/service_names'
 
 const CURATED_CATEGORIES_KEY = 'curated-categories'
 const WIKIPEDIA_STATE_KEY = 'wikipedia-state'
+const CUSTOM_LIBRARIES_KEY = 'custom-libraries'
+
+type CustomLibrary = { id: number; name: string; base_url: string; is_default: boolean }
+type BrowseResult = {
+  directories: { name: string; url: string }[]
+  files: { name: string; url: string; size_bytes: number | null }[]
+}
 
 export default function ZimRemoteExplorer() {
   const queryClient = useQueryClient()
@@ -56,6 +72,20 @@ export default function ZimRemoteExplorer() {
   const [selectedWikipedia, setSelectedWikipedia] = useState(null)
   const [isSubmittingWikipedia, setIsSubmittingWikipedia] = useState(false)
 
+  // Custom library state - persist selection to localStorage
+  const [selectedSource, setSelectedSource] = useState<'default' | number>(() => {
+    try {
+      const saved = localStorage.getItem('nomad:zim-library-source')
+      if (saved && saved !== 'default') return parseInt(saved, 10)
+    } catch {}
+    return 'default'
+  })
+  const [browseUrl, setBrowseUrl] = useState(null)
+  const [breadcrumbs, setBreadcrumbs] = useState<{ name: string; url: string }[]>([])
+  const [manageModalOpen, setManageModalOpen] = useState(false)
+  const [newLibraryName, setNewLibraryName] = useState('')
+  const [newLibraryUrl, setNewLibraryUrl] = useState('')
+
   const debouncedSetQuery = debounce((val: string) => {
     setQuery(val)
   }, 400)
@@ -79,6 +109,26 @@ export default function ZimRemoteExplorer() {
     enabled: true,
   })
 
+  // Fetch custom libraries
+  const { data: customLibraries } = useQuery({
+    queryKey: [CUSTOM_LIBRARIES_KEY],
+    queryFn: () => api.listCustomLibraries(),
+    refetchOnWindowFocus: false,
+  })
+
+  // Browse custom library directory
+  const {
+    data: browseData,
+    isLoading: isBrowsing,
+    error: browseError,
+  } = useQuery({
+    queryKey: ['browse-library', browseUrl],
+    queryFn: () => api.browseLibrary(browseUrl!) as Promise,
+    enabled: !!browseUrl && selectedSource !== 'default',
+    refetchOnWindowFocus: false,
+    retry: false,
+  })
+
   const { data, fetchNextPage, isFetching, isLoading } =
     useInfiniteQuery({
       queryKey: ['remote-zim-files', query],
@@ -97,6 +147,7 @@ export default function ZimRemoteExplorer() {
       getNextPageParam: (lastPage) => (lastPage.has_more ? lastPage.next_start : undefined),
       refetchOnWindowFocus: false,
       placeholderData: keepPreviousData,
+      enabled: selectedSource === 'default',
     })
 
   const flatData = useMemo(() => {
@@ -140,6 +191,50 @@ export default function ZimRemoteExplorer() {
     fetchOnBottomReached(tableParentRef.current)
   }, [fetchOnBottomReached])
 
+  // Restore custom library selection on mount when data loads
+  useEffect(() => {
+    if (selectedSource !== 'default' && customLibraries) {
+      const lib = customLibraries.find((l) => l.id === selectedSource)
+      if (lib && !browseUrl) {
+        setBrowseUrl(lib.base_url)
+        setBreadcrumbs([{ name: lib.name, url: lib.base_url }])
+      } else if (!lib) {
+        // Saved library was deleted
+        setSelectedSource('default')
+        localStorage.setItem('nomad:zim-library-source', 'default')
+      }
+    }
+  }, [customLibraries, selectedSource])
+
+  // When selecting a custom library, navigate to its root
+  const handleSourceChange = (value: string) => {
+    localStorage.setItem('nomad:zim-library-source', value)
+    if (value === 'default') {
+      setSelectedSource('default')
+      setBrowseUrl(null)
+      setBreadcrumbs([])
+    } else {
+      const id = parseInt(value, 10)
+      const lib = customLibraries?.find((l) => l.id === id)
+      if (lib) {
+        setSelectedSource(id)
+        setBrowseUrl(lib.base_url)
+        setBreadcrumbs([{ name: lib.name, url: lib.base_url }])
+      }
+    }
+  }
+
+  const navigateToDirectory = (name: string, url: string) => {
+    setBrowseUrl(url)
+    setBreadcrumbs((prev) => [...prev, { name, url }])
+  }
+
+  const navigateToBreadcrumb = (index: number) => {
+    const crumb = breadcrumbs[index]
+    setBrowseUrl(crumb.url)
+    setBreadcrumbs((prev) => prev.slice(0, index + 1))
+  }
+
   async function confirmDownload(record: RemoteZimFileEntry) {
     openModal(
        {
+          downloadCustomFile(file)
+          closeAllModals()
+        }}
+        onCancel={closeAllModals}
+        open={true}
+        confirmText="Download"
+        cancelText="Cancel"
+        confirmVariant="primary"
+      >
+        

+          Are you sure you want to download{' '}
+          {file.name}
+          {file.size_bytes ? ` (${formatBytes(file.size_bytes)})` : ''}? The Kiwix
+          application will be restarted after the download is complete.
+        

+      ,
+      'confirm-download-custom-modal'
+    )
+  }
+
   async function downloadFile(record: RemoteZimFileEntry) {
     try {
       await api.downloadRemoteZimFile(record.download_url, {
@@ -179,6 +299,26 @@ export default function ZimRemoteExplorer() {
     }
   }
 
+  async function downloadCustomFile(file: { name: string; url: string; size_bytes: number | null }) {
+    try {
+      await api.downloadRemoteZimFile(file.url, {
+        title: file.name.replace(/\.zim$/, ''),
+        size_bytes: file.size_bytes ?? undefined,
+      })
+      addNotification({
+        message: `Started downloading "${file.name}"`,
+        type: 'success',
+      })
+      invalidateDownloads()
+    } catch (error) {
+      console.error('Error downloading file:', error)
+      addNotification({
+        message: 'Failed to start download.',
+        type: 'error',
+      })
+    }
+  }
+
   // Category/tier handlers
   const handleCategoryClick = (category: CategoryWithStatus) => {
     if (!isOnline) return
@@ -264,6 +404,35 @@ export default function ZimRemoteExplorer() {
     },
   })
 
+  // Custom library management
+  const addLibraryMutation = useMutation({
+    mutationFn: () => api.addCustomLibrary(newLibraryName.trim(), newLibraryUrl.trim()),
+    onSuccess: () => {
+      addNotification({ message: 'Custom library added.', type: 'success' })
+      queryClient.invalidateQueries({ queryKey: [CUSTOM_LIBRARIES_KEY] })
+      setNewLibraryName('')
+      setNewLibraryUrl('')
+    },
+    onError: () => {
+      addNotification({ message: 'Failed to add custom library.', type: 'error' })
+    },
+  })
+
+  const removeLibraryMutation = useMutation({
+    mutationFn: (id: number) => api.removeCustomLibrary(id),
+    onSuccess: (_data, id) => {
+      addNotification({ message: 'Custom library removed.', type: 'success' })
+      queryClient.invalidateQueries({ queryKey: [CUSTOM_LIBRARIES_KEY] })
+      if (selectedSource === id) {
+        setSelectedSource('default')
+        setBrowseUrl(null)
+        setBreadcrumbs([])
+      }
+    },
+  })
+
+  const hasCustomLibraries = customLibraries && customLibraries.length > 0
+
   return (
     
       
@@ -302,7 +471,7 @@ export default function ZimRemoteExplorer() {
               Force Refresh Collections
- + {/* Wikipedia Selector */} {isLoadingWikipedia ? (
@@ -360,87 +529,303 @@ export default function ZimRemoteExplorer() { ) : (

No curated content categories available.

)} - -
- { - setQueryUI(e.target.value) - debouncedSetQuery(e.target.value) - }} - className="w-1/3" - leftIcon={} - /> + + {/* Kiwix Library / Custom Library Browser */} +
+ + setManageModalOpen(true)} + disabled={!isOnline} + icon="IconLibrary" + > + {hasCustomLibraries ? 'Manage Custom Libraries' : 'Add Custom Library'} +
- - data={flatData.map((i, idx) => { - const row = virtualizer.getVirtualItems().find((v) => v.index === idx) - return { - ...i, - height: `${row?.size || 48}px`, // Use the size from the virtualizer - translateY: row?.start || 0, - } - })} - ref={tableParentRef} - loading={isLoading} - columns={[ - { - accessor: 'title', - }, - { - accessor: 'author', - }, - { - accessor: 'summary', - }, - { - accessor: 'updated', - render(record) { - return new Intl.DateTimeFormat('en-US', { - dateStyle: 'medium', - }).format(new Date(record.updated)) - }, - }, - { - accessor: 'size_bytes', - title: 'Size', - render(record) { - return formatBytes(record.size_bytes) - }, - }, - { - accessor: 'actions', - render(record) { - return ( -
- { - confirmDownload(record) - }} + + {/* Source selector dropdown */} + {hasCustomLibraries && ( +
+ + +
+ )} + + {/* Default Kiwix library browser */} + {selectedSource === 'default' && ( + <> +
+ { + setQueryUI(e.target.value) + debouncedSetQuery(e.target.value) + }} + className="w-1/3" + leftIcon={} + /> +
+ + data={flatData.map((i, idx) => { + const row = virtualizer.getVirtualItems().find((v) => v.index === idx) + return { + ...i, + height: `${row?.size || 48}px`, + translateY: row?.start || 0, + } + })} + ref={tableParentRef} + loading={isLoading} + columns={[ + { + accessor: 'title', + }, + { + accessor: 'author', + }, + { + accessor: 'summary', + }, + { + accessor: 'updated', + render(record) { + return new Intl.DateTimeFormat('en-US', { + dateStyle: 'medium', + }).format(new Date(record.updated)) + }, + }, + { + accessor: 'size_bytes', + title: 'Size', + render(record) { + return formatBytes(record.size_bytes) + }, + }, + { + accessor: 'actions', + render(record) { + return ( +
+ { + confirmDownload(record) + }} + > + Download + +
+ ) + }, + }, + ]} + className="relative overflow-x-auto overflow-y-auto h-[600px] w-full mt-4" + tableBodyStyle={{ + position: 'relative', + height: `${virtualizer.getTotalSize()}px`, + }} + containerProps={{ + onScroll: (e) => fetchOnBottomReached(e.currentTarget as HTMLDivElement), + }} + compact + rowLines + /> + + )} + + {/* Custom library directory browser */} + {selectedSource !== 'default' && ( +
+ {/* Breadcrumb navigation */} +
- ) - }, - }, - ]} - className="relative overflow-x-auto overflow-y-auto h-[600px] w-full mt-4" - tableBodyStyle={{ - position: 'relative', - height: `${virtualizer.getTotalSize()}px`, - }} - containerProps={{ - onScroll: (e) => fetchOnBottomReached(e.currentTarget as HTMLDivElement), - }} - compact - rowLines - /> + {crumb.name} + + ) : ( + {crumb.name} + )} + + ))} + + + {isBrowsing && ( +
+
+
+ )} + + {browseError && ( + + )} + + {!isBrowsing && !browseError && browseData && ( +
+ {browseData.directories.length === 0 && browseData.files.length === 0 ? ( +

+ No directories or ZIM files found at this location. +

+ ) : ( + + + + + + + + + + {browseData.directories.map((dir) => ( + navigateToDirectory(dir.name, dir.url)} + > + + + + + ))} + {browseData.files.map((file) => ( + + + + + + ))} + +
NameSize
+ + + {dir.name} + + -- + +
+ + + {file.name} + + + {file.size_bytes ? formatBytes(file.size_bytes) : '--'} + + confirmCustomDownload(file)} + > + Download + +
+ )} +
+ )} +
+ )} + + + {/* Manage Custom Libraries Modal */} + setManageModalOpen(false)} + cancelText="Close" + > +
+
+

+ Add Kiwix mirrors or other ZIM file sources for faster downloads. +

+ + {/* Existing libraries */} + {customLibraries && customLibraries.length > 0 && ( +
+ {customLibraries.map((lib) => ( +
+
+

+ {lib.name} + {lib.is_default && ( + (built-in) + )} +

+

{lib.base_url}

+
+ {!lib.is_default && ( + + )} +
+ ))} +
+ )} + + {/* Add new library form */} +
+ setNewLibraryName(e.target.value)} + /> + setNewLibraryUrl(e.target.value)} + /> + addLibraryMutation.mutate()} + disabled={ + !newLibraryName.trim() || + !newLibraryUrl.trim() || + addLibraryMutation.isPending + } + > + Add Library + +
+
+
+
diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 724734d..fb0d47d 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -183,6 +183,12 @@ router router.get('/wikipedia', [ZimController, 'getWikipediaState']) router.post('/wikipedia/select', [ZimController, 'selectWikipedia']) + + router.get('/custom-libraries', [ZimController, 'listCustomLibraries']) + router.post('/custom-libraries', [ZimController, 'addCustomLibrary']) + router.delete('/custom-libraries/:id', [ZimController, 'removeCustomLibrary']) + router.get('/browse-library', [ZimController, 'browseLibrary']) + router.delete('/:filename', [ZimController, 'delete']) }) .prefix('/api/zim') From 0ddcfe901105f3657b8d03f329471edf4ee58378 Mon Sep 17 00:00:00 2001 From: Jake Turner <52841588+jakeaturner@users.noreply.github.com> Date: Mon, 4 May 2026 11:54:56 -0700 Subject: [PATCH 13/18] fix(System): self-heal stale updateAvailable flag after sidecar-driven update (#825) --- admin/adonisrc.ts | 1 + admin/providers/version_check_provider.ts | 56 +++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 admin/providers/version_check_provider.ts diff --git a/admin/adonisrc.ts b/admin/adonisrc.ts index 34586ca..741b160 100644 --- a/admin/adonisrc.ts +++ b/admin/adonisrc.ts @@ -56,6 +56,7 @@ export default defineConfig({ () => import('#providers/map_static_provider'), () => import('#providers/kiwix_migration_provider'), () => import('#providers/qdrant_restart_policy_provider'), + () => import('#providers/version_check_provider'), ], /* diff --git a/admin/providers/version_check_provider.ts b/admin/providers/version_check_provider.ts new file mode 100644 index 0000000..20add77 --- /dev/null +++ b/admin/providers/version_check_provider.ts @@ -0,0 +1,56 @@ +import logger from '@adonisjs/core/services/logger' +import type { ApplicationService } from '@adonisjs/core/types' + +/** + * Self-heals stale `system.updateAvailable` after a sidecar-driven update. + * + * When the admin container is recreated on a new image, the KVStore still + * carries pre-update values for `system.updateAvailable` and + * `system.latestVersion`. Without intervention the UI keeps showing the + * "update available" banner until the next scheduled CheckUpdateJob (could be up to ~12h). + * + * Synchronous self-heal (no network): if the cached "latest" is not newer + * than the version we are now running, clear `updateAvailable`. The next + * scheduled CheckUpdateJob refreshes the cache from GitHub — we deliberately + * do not hit the network from boot to avoid coupling container startup to + * a network request to Github (e.g. container restart loop = flooding GitHub with requests). + * + * Note: this provider does not set `updateAvailable` to true if the cached + * "latest" is newer than the current version. We rely on the next scheduled + * CheckUpdateJob to do that, to avoid false positives in case of a stale cache. + */ +export default class VersionCheckProvider { + constructor(protected app: ApplicationService) { } + + async boot() { + if (this.app.getEnvironment() !== 'web') return + + setImmediate(async () => { + try { + const KVStore = (await import('#models/kv_store')).default + const { SystemService } = await import('#services/system_service') + const { isNewerVersion } = await import('../app/utils/version.js') + + const current = SystemService.getAppVersion() + if (current === 'dev' || current === '0.0.0'){ + logger.info(`[VersionCheckProvider] Skipping self-heal for version ${current}. Appears to be a dev build without proper version set.`) + return + } + + logger.info(`[VersionCheckProvider] Checking for stale updateAvailable (current=${current})`) + + const cachedLatest = (await KVStore.getValue('system.latestVersion')) as string | null + const earlyAccess = ((await KVStore.getValue('system.earlyAccess')) ?? false) as boolean + + if (cachedLatest && !isNewerVersion(cachedLatest, current, earlyAccess)) { + await KVStore.setValue('system.updateAvailable', false) + logger.info( + `[VersionCheckProvider] Cleared stale updateAvailable (cached=${cachedLatest}, current=${current})` + ) + } + } catch (err: any) { + logger.warn(`[VersionCheckProvider] Self-heal skipped: ${err?.message ?? err}`) + } + }) + } +} From 0fdf31c2e42874e8f7db2b537a487a9eced113ed Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Mon, 4 May 2026 19:21:51 +0000 Subject: [PATCH 14/18] docs: update release notes --- admin/docs/release-notes.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/admin/docs/release-notes.md b/admin/docs/release-notes.md index c8c2766..cc7b654 100644 --- a/admin/docs/release-notes.md +++ b/admin/docs/release-notes.md @@ -2,21 +2,57 @@ ## Unreleased +### Features +- **AI Assistant**: Added improved support for AMD GPU acceleration for Ollama via ROCm + HSA override. Thanks @chriscrosstalk for the contribution! +- **Content Explorer**: Added support for custom ZIM library sources and pre-seeded ZIM library mirrors in addition to the default Kiwix library. Thanks @chriscrosstalk for the contribution! +- **Content Manager**: Content update sizes and downloads are now properly displayed in Active Downloads with progress bars and friendly names. Thanks @chriscrosstalk for the contribution! +- **Maps**: Map regions can now be extracted and downloaded locally from PMTiles to avoid the need for a full global map download for users who only want specific regions. Thanks @bgauger for the contribution! + +### Bug Fixes +- **API**: Compression is now skipped for Server-Sent Events (SSE) responses to prevent issues with streaming endpoints. Thanks @chriscrosstalk for the fix! +- **Maps**: Fixed logic issues with the global map banner display. Thanks @Gujiassh for the fix! +- **Maps**: The selected map file is now properly deleted after confirming the action in the UI. Thanks @cuyua9 for the fix! +- **System**: Fixed an issue where the a pending update could still be indicated in the UI even after the system was updated successfully. Thanks @jakeaturner for the fix! + +### Improvements +- **Build**: The Command Center image now uses the VERSION build arg to write `app/version.json` with the current version for improved version tracking and debugging, even in RC environments. Thanks @chriscrosstalk for the contribution! +- **Content Manager**: Added a sortable file size column to the ZIM files table in the Content Manager for easier management of storage space. Thanks @chriscrosstalk for the contribution! +- **Dependencies**: All package.json dependencies have been pinned to specific versions to ensure stability and reduce the risk of unexpected breaking changes/supply-chain compromises from upstream packages. Thanks @jakeaturner for the contribution! +- **Dependencies**: Updated various dependencies to close security vulnerabilities and improve stability +- **Docs**: Update CONTIRBUTING.md to require an issue to be opened before submitting a PR for non-trivial changes to ensure proper discussion and review of proposed changes. Thanks @chriscrosstalk for the contribution! +- **Docs**: Added the map markers endpoints to the API reference documentation. Thanks @kennethbrewer3 for the contribution! +- **Docs**: Added a link to the new WSL2 install guide in the README and FAQ. Thanks @chriscrosstalk for the contribution! +- **Install**: The install script now warns loudly if the user is attempting to install on a non-x86_64/amd64 platform to prevent unsupported installations and potential issues. Thanks @chriscrosstalk for the contribution! +- **Maps**: The maps API endpoints now properly accept and validate notes, marker_type, and position data for map markers and persist them in the database for retrieval in the UI. Thanks @jrsphoto for the contribution! +- **Maps**: The current coordinates of the mouse pointer can now be displayed in the map viewer for easier navigation and exploration. Thanks @kennethbrewer3 for the contribution! +- **RAG**: NOMAD now properly passed `num_ctx` and truncation to the Ollama embedding endpoint to ensure that the context window of the model is best utilized for embeddings. Thanks @chriscrosstalk for the contribution! +- **RAG**: Added a manual start button for Qdrant and a self-healing mechanism for Qdrant's restart-policy to ensure that the vector database is running properly for embedding and retrieval tasks. Thanks @hestela for the contribution! + +## Version 1.31.1 - April 21, 2026 + ### Features ### Bug Fixes - **AI Assistant**: In-progress model downloads can now be cancelled properly and the progress UI now matches that of file downloads. Thanks @chriscrosstalk for the contribution! - **AI Assistant**: Fixed an issue where the AI Assistant settings page could crash if a model object did not have a details property. Thanks @hestela for the fix! +- **AI Assistant**: Fixed an issue with non-embeddable files being queued for embedding and flooding logs with errors. Thanks @sbruschke for the bug report and @chriscrosstalk for the fix! +- **AI Assistant**: Fixed an issue with ZIM batch embedding using the wrong batch count and causing remaining batches to be skipped. Thanks @sbruschke for the bug report and @chriscrosstalk for the fix! +- **AI Assistant**: Fixed an issue with ZIM content extraction only extracting the first-level children of the article body and thus missing a lot of content. Thanks @sbruschke for the bug report and @chriscrosstalk for the fix! - **Disk Collector**: Improved reporting for NFS mount stats and display in the UI. Thanks @bgauger and @bravosierra99 for the contribution! - **Downloads**: Downloads are now staged to .tmp files and atomically renamed upon completion to prevent issues with incomplete/corrupt files. Thanks @artbird309 for the contribution! - **Downloads**: Removed a duplicate error listener and improved stability when handling Range requests for file downloads. Thanks @jakeaturner for the contribution! - **Downloads**: Added improved handling for corrupt ZIM file downloads and removed duplicate Ollama download logs. Thanks @aegisman for the contribution! - **Security**: Closed a potential SSRF vulnerability in the map file download functionality by implementing stricter URL validation and blocking private IP ranges. Thanks @LuisMIguelFurlanettoSousa for the fix! - **Security**: Sanitized error messages from the backend to prevent potential information disclosure. Thanks @LuisMIguelFurlanettoSousa for the fix! +- **UI**: Fixed an issue with broken pagination for the Content Explorer that could cause some users to see a "No records found" message indefinitely. Thanks @johno10661 for the bug report and @chriscrosstalk for the fix! +- **UI**: Fixed an issue where all storage devices could report as "NAS Storage" regardless of actual type. Thanks @bgauger for the fix! ### Improvements - **AI Assistant**: Now uses the currently loaded model for query rewriting and chat title generation for improved performance and consistency. Thanks @hestela for the contribution! +- **AI Assistant**: When a remote Ollama URL is configured, the Command Center will now attempt to stop NOMAD's local Ollama container to free up resources and avoid confusion. Thanks @chriscrosstalk for the contribution! - **Dependencies**: Updated various dependencies to close security vulnerabilities and improve stability +- **Docs**: Added a "Community Add-Ons" page to the documentation to highlight some of the amazing community contributions that have been made since launch. Thanks @chriscrosstalk for the contribution! +- **Privacy**: Added the appropriate environment variable to disable telemetry for the Qdrant container. Note that this will only take effect on new installations of if the Qdrant container is force re-installed on existing installations. Thanks @berkdamerc for the find and @chriscrosstalk for the contribution! ## Version 1.31.0 - April 3, 2026 From 1ad898bc8b7222961e07cbaf4e1fc67258372e4e Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Tue, 5 May 2026 10:33:18 -0700 Subject: [PATCH 15/18] fix(UI): four fixes for the System Update page (#827) Closes #826. 1. Heading and subtext now read from `versionInfo` state (which the Check Again mutation already populates) instead of the server-rendered `props.system`. Previously the card kept showing "System Up to Date / Your system is running the latest version!" alongside the new `Latest Version` row + Start Update button after a successful recheck. Status icon also switched to `versionInfo` for consistency. 2. The pulling-state heading rendered the lowercase status enum (`pulling`, `pulled`, ...) and relied on a Tailwind `capitalize` class for the visible glyph. Screen readers and other accessible-name consumers got the lowercase value with no transform applied. Replaced with a `STAGE_LABELS` map so visual + accessible names match. 3. The sidecar (install/sidecar-updater/update-watcher.sh) writes `complete` for ~5s, then resets the status file to `idle`. The SPA could miss that window across the admin container restart, leaving the page parked on its last observed progress percentage indefinitely while the upgrade was actually finished on disk. A `seenAdvancedStageRef` now records whether the session ever observed an advanced stage; a later poll seeing `idle` is treated as the missed completion, and the page reloads as advertised in step 3 of the on-screen process. Reset on each Start Update. 4. Toggling Enable Early Access now triggers a recheck on success, so the eligible-version list updates immediately instead of requiring a manual Check Again click. Single file touched: admin/inertia/pages/settings/update.tsx. Typecheck (tsc --noEmit) passes; static UI changes verified in source. Co-authored-by: Claude Opus 4.7 (1M context) --- admin/inertia/pages/settings/update.tsx | 56 +++++++++++++++++++++---- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/admin/inertia/pages/settings/update.tsx b/admin/inertia/pages/settings/update.tsx index 348040d..06db3bb 100644 --- a/admin/inertia/pages/settings/update.tsx +++ b/admin/inertia/pages/settings/update.tsx @@ -5,7 +5,7 @@ import StyledTable from '~/components/StyledTable' import StyledSectionHeader from '~/components/StyledSectionHeader' import ActiveDownloads from '~/components/ActiveDownloads' import Alert from '~/components/Alert' -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { IconAlertCircle, IconArrowBigUpLines, IconCheck, IconCircleCheck, IconReload } from '@tabler/icons-react' import { SystemUpdateStatus } from '../../../types/system' import type { ContentUpdateCheckResult, ResourceUpdateInfo } from '../../../types/collections' @@ -24,6 +24,23 @@ type Props = { earlyAccess: boolean } +const STAGE_LABELS: Record = { + idle: 'Preparing Update', + starting: 'Starting Update', + pulling: 'Pulling Images', + pulled: 'Images Pulled', + recreating: 'Recreating Containers', + complete: 'Update Complete', + error: 'Update Failed', +} + +const ADVANCED_STAGES: ReadonlySet = new Set([ + 'pulling', + 'pulled', + 'recreating', + 'complete', +]) + function ContentUpdatesSection() { const { addNotification } = useNotifications() const queryClient = useQueryClient() @@ -251,6 +268,12 @@ export default function SystemUpdatePage(props: { system: Props }) { const [email, setEmail] = useState('') const [versionInfo, setVersionInfo] = useState>(props.system) const [showConnectionLostNotice, setShowConnectionLostNotice] = useState(false) + // Tracks whether this update session has progressed past 'idle'/'starting'. + // The sidecar sits on 'complete' for ~5s before resetting to 'idle' (see + // install/sidecar-updater/update-watcher.sh), and the SPA can miss that + // window across the admin container restart. If we resurface to 'idle' + // after seeing an advanced stage, treat it as the missed completion. + const seenAdvancedStageRef = useRef(false) const earlyAccessSetting = useSystemSetting({ key: 'system.earlyAccess', initialData: { @@ -270,11 +293,22 @@ export default function SystemUpdatePage(props: { system: Props }) { } setUpdateStatus(response) + if (ADVANCED_STAGES.has(response.stage)) { + seenAdvancedStageRef.current = true + } + // If we can connect again, hide the connection lost notice setShowConnectionLostNotice(false) - // Check if update is complete or errored - if (response.stage === 'complete') { + // Check if update is complete or errored. We also treat a return to + // 'idle' as completion if we previously saw an advanced stage — this + // catches the race where the sidecar's brief 'complete' window passes + // while we're disconnected during the admin container restart. + const isComplete = + response.stage === 'complete' || + (response.stage === 'idle' && seenAdvancedStageRef.current) + + if (isComplete) { // Re-check version so the KV store clears the stale "update available" flag // before we reload, otherwise the banner shows "current → current" try { @@ -304,6 +338,7 @@ export default function SystemUpdatePage(props: { system: Props }) { const handleStartUpdate = async () => { try { setError(null) + seenAdvancedStageRef.current = false setIsUpdating(true) const response = await api.startSystemUpdate() if (!response || !response.success) { @@ -368,7 +403,7 @@ export default function SystemUpdatePage(props: { system: Props }) { if (updateStatus?.stage === 'error') return if (isUpdating) return - if (props.system.updateAvailable) + if (versionInfo.updateAvailable) return return } @@ -380,6 +415,9 @@ export default function SystemUpdatePage(props: { system: Props }) { onSuccess: () => { addNotification({ message: 'Setting updated successfully.', type: 'success' }) earlyAccessSetting.refetch() + // Toggling Early Access changes which versions are eligible, so re-evaluate + // immediately rather than making the user click Check Again. + checkVersionMutation.mutate() }, onError: (error) => { console.error('Error updating setting:', error) @@ -461,11 +499,11 @@ export default function SystemUpdatePage(props: { system: Props }) { {!isUpdating && ( <>

- {props.system.updateAvailable ? 'Update Available' : 'System Up to Date'} + {versionInfo.updateAvailable ? 'Update Available' : 'System Up to Date'}

- {props.system.updateAvailable - ? `A new version (${props.system.latestVersion}) is available for your Project N.O.M.A.D. instance.` + {versionInfo.updateAvailable + ? `A new version (${versionInfo.latestVersion}) is available for your Project N.O.M.A.D. instance.` : 'Your system is running the latest version!'}

@@ -473,8 +511,8 @@ export default function SystemUpdatePage(props: { system: Props }) { {isUpdating && updateStatus && ( <> -

- {updateStatus.stage === 'idle' ? 'Preparing Update' : updateStatus.stage} +

+ {STAGE_LABELS[updateStatus.stage] ?? updateStatus.stage}

{updateStatus.message}

From cb47e1e4e35a9ffa7f11f168f716802911b10194 Mon Sep 17 00:00:00 2001 From: cosmistack-bot Date: Mon, 4 May 2026 19:32:01 +0000 Subject: [PATCH 16/18] chore(release): 1.32.0-rc.1 [skip ci] --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2a8ff50..86b418c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "project-nomad", - "version": "1.31.0", + "version": "1.32.0-rc.1", "description": "\"", "main": "index.js", "scripts": { From 03ab614f9961eb9c8ab968fdaf5540a8723fac11 Mon Sep 17 00:00:00 2001 From: Ben Gauger Date: Mon, 4 May 2026 14:39:10 -0600 Subject: [PATCH 17/18] fix(Maps): send filename instead of full path to delete endpoint --- admin/inertia/pages/settings/maps.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/inertia/pages/settings/maps.tsx b/admin/inertia/pages/settings/maps.tsx index f95fa56..d2df0f3 100644 --- a/admin/inertia/pages/settings/maps.tsx +++ b/admin/inertia/pages/settings/maps.tsx @@ -138,7 +138,7 @@ export default function MapsManager(props: { try { setDeletingFileKey(file.key) - await api.deleteMapRegionFile(file.key) + await api.deleteMapRegionFile(file.name) addNotification({ type: 'success', message: `${file.name} has been deleted.`, From 63282565a958648064b0fbe35bf2a54ee9d1cad4 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Mon, 4 May 2026 15:41:23 -0700 Subject: [PATCH 18/18] fix(Maps): render notes in marker popup when populated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #796. The maps API has accepted and persisted `notes` on map markers since PR #770, but the marker popup component still rendered name only and ignored the field. Now the popup shows a notes block beneath the name when it's populated, with whitespace preserved and long text wrapped. Threaded `notes` through the read path: - `api.listMapMarkers` / `api.createMapMarker` response types - `MapMarker` interface in `useMapMarkers` and the data.map projection - `MapComponent`'s selectedMarker popup The create/update UI is unchanged — users still set notes via the API or DB directly, matching the issue's stated scope. A marker entry with empty/whitespace-only notes renders the same as before. Co-Authored-By: Claude Opus 4.7 (1M context) --- admin/inertia/components/maps/MapComponent.tsx | 5 +++++ admin/inertia/hooks/useMapMarkers.ts | 3 +++ admin/inertia/lib/api.ts | 4 ++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/admin/inertia/components/maps/MapComponent.tsx b/admin/inertia/components/maps/MapComponent.tsx index 07fe1c6..f7a3a8a 100644 --- a/admin/inertia/components/maps/MapComponent.tsx +++ b/admin/inertia/components/maps/MapComponent.tsx @@ -243,6 +243,11 @@ export default function MapComponent({ closeOnClick={false} >
{selectedMarker.name}
+ {selectedMarker.notes && selectedMarker.notes.trim() && ( +
+ {selectedMarker.notes} +
+ )} )} diff --git a/admin/inertia/hooks/useMapMarkers.ts b/admin/inertia/hooks/useMapMarkers.ts index ba999eb..a0e818e 100644 --- a/admin/inertia/hooks/useMapMarkers.ts +++ b/admin/inertia/hooks/useMapMarkers.ts @@ -18,6 +18,7 @@ export interface MapMarker { longitude: number latitude: number color: PinColorId + notes: string | null createdAt: string } @@ -36,6 +37,7 @@ export function useMapMarkers() { longitude: m.longitude, latitude: m.latitude, color: m.color as PinColorId, + notes: m.notes ?? null, createdAt: m.created_at, })) ) @@ -54,6 +56,7 @@ export function useMapMarkers() { longitude: result.longitude, latitude: result.latitude, color: result.color as PinColorId, + notes: result.notes ?? null, createdAt: result.created_at, } setMarkers((prev) => [...prev, marker]) diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 3e015d8..aa1369b 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -631,7 +631,7 @@ class API { async listMapMarkers() { return catchInternal(async () => { const response = await this.client.get< - Array<{ id: number; name: string; longitude: number; latitude: number; color: string; created_at: string }> + Array<{ id: number; name: string; longitude: number; latitude: number; color: string; notes: string | null; created_at: string }> >('/maps/markers') return response.data })() @@ -640,7 +640,7 @@ class API { async createMapMarker(data: { name: string; longitude: number; latitude: number; color?: string }) { return catchInternal(async () => { const response = await this.client.post< - { id: number; name: string; longitude: number; latitude: number; color: string; created_at: string } + { id: number; name: string; longitude: number; latitude: number; color: string; notes: string | null; created_at: string } >('/maps/markers', data) return response.data })()