merged in dev and fixed conflicts
This commit is contained in:
Kenneth Brewer
commit
a452366078
34
Dockerfile
34
Dockerfile
|
|
@ -1,7 +1,14 @@
|
|||
FROM node:22-slim AS base
|
||||
|
||||
# Install bash & curl for entrypoint script compatibility, graphicsmagick for pdf2pic, and vips-dev & build-base for sharp
|
||||
RUN apt-get update && apt-get install -y bash curl graphicsmagick libvips-dev build-essential
|
||||
RUN apt-get update && apt-get install -y \
|
||||
bash \
|
||||
curl \
|
||||
graphicsmagick \
|
||||
libvips-dev \
|
||||
build-essential \
|
||||
pciutils \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# All deps stage
|
||||
FROM base AS deps
|
||||
|
|
@ -27,6 +34,31 @@ FROM base
|
|||
ARG VERSION=dev
|
||||
ARG BUILD_DATE
|
||||
ARG VCS_REF
|
||||
ARG TARGETARCH
|
||||
|
||||
# go-pmtiles (regional map extracts). Pinned so the CLI's stdout format stays
|
||||
# in sync with parseDryRunOutput().
|
||||
ARG PMTILES_VERSION=1.30.2
|
||||
# Upstream releases don't ship a checksums file, so pin per-arch SHA256 here.
|
||||
# When bumping PMTILES_VERSION, regenerate these with:
|
||||
# curl -fsSL <release-url> | sha256sum
|
||||
ARG PMTILES_SHA256_AMD64=2cd3aa18868297fc88425038f794efdc0995e0275f4ca16fa496dd79e245a40c
|
||||
ARG PMTILES_SHA256_ARM64=804cdf071834e1156af554c1a26cc42b56b9cde5a2db9c6e3653d16fb846d5fa
|
||||
RUN set -eux; \
|
||||
case "${TARGETARCH:-amd64}" in \
|
||||
amd64) PMTILES_ARCH=x86_64; PMTILES_SHA256="${PMTILES_SHA256_AMD64}" ;; \
|
||||
arm64) PMTILES_ARCH=arm64; PMTILES_SHA256="${PMTILES_SHA256_ARM64}" ;; \
|
||||
*) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
|
||||
esac; \
|
||||
TARBALL="go-pmtiles_${PMTILES_VERSION}_Linux_${PMTILES_ARCH}.tar.gz"; \
|
||||
cd /tmp; \
|
||||
curl -fsSL -o "$TARBALL" \
|
||||
"https://github.com/protomaps/go-pmtiles/releases/download/v${PMTILES_VERSION}/${TARBALL}"; \
|
||||
echo "${PMTILES_SHA256} ${TARBALL}" | sha256sum -c -; \
|
||||
tar -xzf "$TARBALL" -C /usr/local/bin pmtiles; \
|
||||
rm -f "$TARBALL"; \
|
||||
chmod +x /usr/local/bin/pmtiles; \
|
||||
/usr/local/bin/pmtiles version
|
||||
|
||||
# Labels
|
||||
LABEL org.opencontainers.image.title="Project N.O.M.A.D" \
|
||||
|
|
|
|||
6
FAQ.md
6
FAQ.md
|
|
@ -20,7 +20,9 @@ Long answer: Custom storage paths, mount points, and external drives (like iSCSI
|
|||
|
||||
## Can I run NOMAD on MAC, WSL2, or a non-Debian-based Distro?
|
||||
|
||||
See [Why does NOMAD require a Debian-based OS?](#why-does-nomad-require-a-debian-based-os)
|
||||
**WSL2 on Windows** is community-supported via the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) — covers two install paths (native Docker and Docker Desktop) with all known gotchas documented and empirical performance numbers comparing WSL2 to bare-metal.
|
||||
|
||||
**macOS and other non-Debian Linux distros** aren't officially supported. See [Why does NOMAD require a Debian-based OS?](#why-does-nomad-require-a-debian-based-os) for details.
|
||||
|
||||
## Why does NOMAD require a Debian-based OS?
|
||||
|
||||
|
|
@ -28,7 +30,7 @@ Project N.O.M.A.D. is currently designed to run on Debian-based Linux distributi
|
|||
|
||||
Support for other operating systems will come in the future, but because our development resources are limited as a free and open-source project, we needed to prioritize our efforts and focus on a narrower set of supported platforms for the initial release. We chose Debian-based Linux as our starting point because it's widely used, easy to spin up, and provides a stable environment for running Docker containers.
|
||||
|
||||
Community members have provided guides for running N.O.M.A.D. on other platforms (e.g. WSL2, Mac, etc.) in our Discord community and [Github Discussions](https://github.com/Crosstalk-Solutions/project-nomad/discussions), so if you're interested in running N.O.M.A.D. on a non-Debian-based system, we recommend checking there for any available resources or guides. However, keep in mind that if you choose to run N.O.M.A.D. on a non-Debian-based system, you may encounter issues that we won't be able to provide support for, and you may need to have a higher level of technical expertise to troubleshoot and resolve any problems that arise.
|
||||
For Windows users, the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) provides a community-supported path. Community members have also published guides for other platforms (e.g. macOS) in our Discord community and [Github Discussions](https://github.com/Crosstalk-Solutions/project-nomad/discussions), so if you're interested in running N.O.M.A.D. on a non-Debian-based system, we recommend checking there for any available resources or guides. However, keep in mind that if you choose to run N.O.M.A.D. on a non-Debian-based system, you may encounter issues that we won't be able to provide support for, and you may need to have a higher level of technical expertise to troubleshoot and resolve any problems that arise.
|
||||
|
||||
## Can I run NOMAD on a Raspberry Pi or other ARM-based device?
|
||||
Project N.O.M.A.D. is currently designed to run on x86-64 architecture, and we have not yet tested or optimized it for ARM-based devices like the Raspberry Pi (and have not published any official images for ARM architecture).
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ sudo bash install_nomad.sh
|
|||
|
||||
Project N.O.M.A.D. is now installed on your device! Open a browser and navigate to `http://localhost:8080` (or `http://DEVICE_IP:8080`) to start exploring!
|
||||
|
||||
For a complete step-by-step walkthrough (including Ubuntu installation), see the [Installation Guide](https://www.projectnomad.us/install).
|
||||
For a complete step-by-step walkthrough (including Ubuntu installation), see the [Installation Guide](https://www.projectnomad.us/install). For Windows users, see the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) — community-supported path covering native Docker and Docker Desktop install routes.
|
||||
|
||||
### Advanced Installation
|
||||
For more control over the installation process, copy and paste the [Docker Compose template](https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/install/management_compose.yaml) into a `docker-compose.yml` file and customize it to your liking (be sure to replace any placeholders with your actual values). Then, run `docker compose up -d` to start the Command Center and its dependencies. Note: this method is recommended for advanced users only, as it requires familiarity with Docker and manual configuration before starting.
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ export default defineConfig({
|
|||
() => import('#providers/map_static_provider'),
|
||||
() => import('#providers/kiwix_migration_provider'),
|
||||
() => import('#providers/qdrant_restart_policy_provider'),
|
||||
() => import('#providers/version_check_provider'),
|
||||
],
|
||||
|
||||
/*
|
||||
|
|
@ -107,6 +108,10 @@ export default defineConfig({
|
|||
pattern: 'resources/views/**/*.edge',
|
||||
reloadServer: false,
|
||||
},
|
||||
{
|
||||
pattern: 'resources/geodata/**/*.geojson',
|
||||
reloadServer: false,
|
||||
},
|
||||
{
|
||||
pattern: 'public/**',
|
||||
reloadServer: false,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ import {
|
|||
assertNotPrivateUrl,
|
||||
downloadCollectionValidator,
|
||||
filenameParamValidator,
|
||||
mapExtractPreflightValidator,
|
||||
mapExtractValidator,
|
||||
remoteDownloadValidator,
|
||||
remoteDownloadValidatorOptional,
|
||||
} from '#validators/common'
|
||||
|
|
@ -87,6 +89,28 @@ export default class MapsController {
|
|||
}
|
||||
}
|
||||
|
||||
async listCountries({}: HttpContext) {
|
||||
return { countries: await this.mapService.listCountries() }
|
||||
}
|
||||
|
||||
async listCountryGroups({}: HttpContext) {
|
||||
return { groups: await this.mapService.listCountryGroups() }
|
||||
}
|
||||
|
||||
async extractPreflight({ request }: HttpContext) {
|
||||
const payload = await request.validateUsing(mapExtractPreflightValidator)
|
||||
return await this.mapService.extractPreflight(payload)
|
||||
}
|
||||
|
||||
async extractRegion({ request }: HttpContext) {
|
||||
const payload = await request.validateUsing(mapExtractValidator)
|
||||
const result = await this.mapService.extractRegion(payload)
|
||||
return {
|
||||
message: 'Extract started successfully',
|
||||
...result,
|
||||
}
|
||||
}
|
||||
|
||||
async styles({ request, response }: HttpContext) {
|
||||
// Automatically ensure base assets are present before generating styles
|
||||
const baseAssetsExist = await this.mapService.ensureBaseAssets()
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import {
|
|||
remoteDownloadWithMetadataValidator,
|
||||
selectWikipediaValidator,
|
||||
} from '#validators/common'
|
||||
import { listRemoteZimValidator } from '#validators/zim'
|
||||
import { addCustomLibraryValidator, browseLibraryValidator, idParamValidator, listRemoteZimValidator } from '#validators/zim'
|
||||
import { inject } from '@adonisjs/core'
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
|
||||
|
|
@ -85,4 +85,51 @@ export default class ZimController {
|
|||
const payload = await request.validateUsing(selectWikipediaValidator)
|
||||
return this.zimService.selectWikipedia(payload.optionId)
|
||||
}
|
||||
|
||||
// Custom library endpoints
|
||||
|
||||
async listCustomLibraries({}: HttpContext) {
|
||||
return this.zimService.listCustomLibraries()
|
||||
}
|
||||
|
||||
async addCustomLibrary({ request, response }: HttpContext) {
|
||||
const payload = await request.validateUsing(addCustomLibraryValidator)
|
||||
assertNotPrivateUrl(payload.base_url)
|
||||
try {
|
||||
const source = await this.zimService.addCustomLibrary(payload.name, payload.base_url)
|
||||
return { message: 'Custom library added', library: source }
|
||||
} catch (error) {
|
||||
if (error.message === 'Maximum of 10 custom libraries allowed') {
|
||||
return response.status(400).send({ message: error.message })
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async removeCustomLibrary({ request, response }: HttpContext) {
|
||||
const payload = await request.validateUsing(idParamValidator)
|
||||
try {
|
||||
await this.zimService.removeCustomLibrary(payload.params.id)
|
||||
return { message: 'Custom library removed' }
|
||||
} catch (error) {
|
||||
if (error.message === 'Custom library not found') {
|
||||
return response.status(404).send({ message: error.message })
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async browseLibrary({ request, response }: HttpContext) {
|
||||
const payload = await request.validateUsing(browseLibraryValidator)
|
||||
try {
|
||||
return await this.zimService.browseLibraryUrl(payload.url)
|
||||
} catch (error) {
|
||||
if (error.message?.includes('loopback or link-local')) {
|
||||
return response.status(400).send({ message: error.message })
|
||||
}
|
||||
return response.status(502).send({
|
||||
message: 'Could not fetch directory listing from the provided URL',
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,294 @@
|
|||
import { Job, UnrecoverableError } from 'bullmq'
|
||||
import { spawn, ChildProcess } from 'child_process'
|
||||
import { createHash } from 'crypto'
|
||||
import { readdir, stat } from 'fs/promises'
|
||||
import { basename, dirname, join } from 'path'
|
||||
import { QueueService } from '#services/queue_service'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { DownloadProgressData } from '../../types/downloads.js'
|
||||
import { PMTILES_BINARY_PATH, buildPmtilesExtractArgs } from '../../constants/map_regions.js'
|
||||
import { deleteFileIfExists } from '../utils/fs.js'
|
||||
|
||||
export interface RunExtractPmtilesJobParams {
|
||||
sourceUrl: string
|
||||
outputFilepath: string
|
||||
/** Path to a GeoJSON FeatureCollection file passed to `pmtiles extract --region`. */
|
||||
regionFilepath: string
|
||||
maxzoom?: number
|
||||
/** Hint for progress reporting; obtained from `pmtiles extract --dry-run` preflight */
|
||||
estimatedBytes?: number
|
||||
filetype: 'map'
|
||||
title?: string
|
||||
resourceMetadata?: {
|
||||
resource_id: string
|
||||
version: string
|
||||
collection_ref: string | null
|
||||
}
|
||||
}
|
||||
|
||||
export class RunExtractPmtilesJob {
|
||||
static get queue() {
|
||||
return 'pmtiles-extract'
|
||||
}
|
||||
|
||||
static get key() {
|
||||
return 'run-pmtiles-extract'
|
||||
}
|
||||
|
||||
/** In-memory registry of active child processes so in-process cancels can SIGTERM them */
|
||||
static childProcesses: Map<string, ChildProcess> = new Map()
|
||||
|
||||
static getJobId(sourceUrl: string, regionFilepath: string, maxzoom?: number): string {
|
||||
const payload = JSON.stringify({ sourceUrl, regionFilepath, maxzoom: maxzoom ?? null })
|
||||
return createHash('sha256').update(payload).digest('hex').slice(0, 16)
|
||||
}
|
||||
|
||||
/** Redis key used to signal cancellation across processes */
|
||||
static cancelKey(jobId: string): string {
|
||||
return `nomad:download:pmtiles-cancel:${jobId}`
|
||||
}
|
||||
|
||||
static async signalCancel(jobId: string): Promise<void> {
|
||||
const queueService = new QueueService()
|
||||
const queue = queueService.getQueue(this.queue)
|
||||
const client = await queue.client
|
||||
await client.set(this.cancelKey(jobId), '1', 'EX', 300)
|
||||
}
|
||||
|
||||
/** Awaits job.updateProgress and swallows BullMQ stale-job errors (code -1),
|
||||
* which occur when the job was removed from Redis (e.g. cancelled) between
|
||||
* the await being issued and the Redis write completing. Anything else
|
||||
* re-throws so it's caught by the surrounding try rather than becoming an
|
||||
* unhandled rejection. */
|
||||
private async safeUpdateProgress(job: Job, progress: DownloadProgressData): Promise<void> {
|
||||
try {
|
||||
await job.updateProgress(progress)
|
||||
} catch (err: any) {
|
||||
if (err?.code !== -1) throw err
|
||||
}
|
||||
}
|
||||
|
||||
async handle(job: Job) {
|
||||
const params = job.data as RunExtractPmtilesJobParams
|
||||
const { sourceUrl, outputFilepath, regionFilepath, maxzoom, estimatedBytes } = params
|
||||
|
||||
logger.info(
|
||||
`[RunExtractPmtilesJob] Starting extract: source=${sourceUrl} region=${regionFilepath} ` +
|
||||
`maxzoom=${maxzoom ?? 'source-max'} out=${outputFilepath}`
|
||||
)
|
||||
|
||||
const queueService = new QueueService()
|
||||
const cancelRedis = await queueService.getQueue(RunExtractPmtilesJob.queue).client
|
||||
|
||||
let userCancelled = false
|
||||
let proc: ChildProcess | null = null
|
||||
let lastReportedBytes = -1
|
||||
|
||||
// One 2s tick polls the Redis cancel signal and reads file-size for progress. pmtiles
|
||||
// writes incrementally but rewrites directories near the end so progress isn't strictly
|
||||
// monotonic — we cap at 99% and skip emit when bytes are unchanged to avoid Redis chatter.
|
||||
const tick = setInterval(async () => {
|
||||
try {
|
||||
const val = await cancelRedis.get(RunExtractPmtilesJob.cancelKey(job.id!))
|
||||
if (val) {
|
||||
await cancelRedis.del(RunExtractPmtilesJob.cancelKey(job.id!))
|
||||
userCancelled = true
|
||||
proc?.kill('SIGTERM')
|
||||
}
|
||||
} catch {
|
||||
// Redis errors non-fatal — in-memory handle also covers same-process cancels
|
||||
}
|
||||
|
||||
try {
|
||||
const fileStat = await stat(outputFilepath)
|
||||
const downloadedBytes = Number(fileStat.size)
|
||||
if (downloadedBytes === lastReportedBytes) return
|
||||
lastReportedBytes = downloadedBytes
|
||||
|
||||
const totalBytes = estimatedBytes ?? 0
|
||||
const percent =
|
||||
totalBytes > 0 ? Math.min(99, Math.floor((downloadedBytes / totalBytes) * 100)) : 0
|
||||
|
||||
await this.safeUpdateProgress(job, {
|
||||
percent,
|
||||
downloadedBytes,
|
||||
totalBytes,
|
||||
lastProgressTime: Date.now(),
|
||||
} as DownloadProgressData)
|
||||
} catch {
|
||||
// File doesn't exist yet (subprocess still setting up)
|
||||
}
|
||||
}, 2000)
|
||||
|
||||
try {
|
||||
const args = buildPmtilesExtractArgs({
|
||||
sourceUrl,
|
||||
outputFilepath,
|
||||
regionFilepath,
|
||||
maxzoom,
|
||||
downloadThreads: 8,
|
||||
overfetch: 0.2,
|
||||
})
|
||||
proc = spawn(PMTILES_BINARY_PATH, args, { stdio: ['ignore', 'pipe', 'pipe'] })
|
||||
RunExtractPmtilesJob.childProcesses.set(job.id!, proc)
|
||||
|
||||
proc.stdout?.on('data', (chunk) => {
|
||||
logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`)
|
||||
})
|
||||
proc.stderr?.on('data', (chunk) => {
|
||||
logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`)
|
||||
})
|
||||
|
||||
const exitCode: number = await new Promise((resolve, reject) => {
|
||||
proc!.on('close', (code) => resolve(code ?? -1))
|
||||
proc!.on('error', (err) => reject(err))
|
||||
})
|
||||
|
||||
if (exitCode !== 0) {
|
||||
await deleteFileIfExists(outputFilepath)
|
||||
if (userCancelled) {
|
||||
throw new UnrecoverableError(`Extract cancelled by user (exit ${exitCode})`)
|
||||
}
|
||||
throw new Error(`pmtiles extract exited with code ${exitCode}`)
|
||||
}
|
||||
|
||||
// Final progress bump — tick caps at 99 so the UI doesn't flicker to 100 mid-extract
|
||||
const finalStat = await stat(outputFilepath)
|
||||
await this.safeUpdateProgress(job, {
|
||||
percent: 100,
|
||||
downloadedBytes: Number(finalStat.size),
|
||||
totalBytes: estimatedBytes ?? Number(finalStat.size),
|
||||
lastProgressTime: Date.now(),
|
||||
} as DownloadProgressData)
|
||||
|
||||
// Reuse the HTTP download path's post-download hook so the file is registered and
|
||||
// the previous version (if any) is deleted
|
||||
await this.onComplete(params)
|
||||
|
||||
logger.info(
|
||||
`[RunExtractPmtilesJob] Completed extract: out=${outputFilepath} size=${finalStat.size} bytes`
|
||||
)
|
||||
|
||||
return { sourceUrl, outputFilepath }
|
||||
} catch (error: any) {
|
||||
if (userCancelled && !(error instanceof UnrecoverableError)) {
|
||||
throw new UnrecoverableError(`Extract cancelled: ${error.message ?? error}`)
|
||||
}
|
||||
throw error
|
||||
} finally {
|
||||
clearInterval(tick)
|
||||
RunExtractPmtilesJob.childProcesses.delete(job.id!)
|
||||
}
|
||||
}
|
||||
|
||||
private async onComplete(params: RunExtractPmtilesJobParams) {
|
||||
if (!params.resourceMetadata) return
|
||||
|
||||
const [{ default: InstalledResource }, { DateTime }, fsUtils] = await Promise.all([
|
||||
import('#models/installed_resource'),
|
||||
import('luxon'),
|
||||
import('../utils/fs.js'),
|
||||
])
|
||||
|
||||
const fileStat = await fsUtils.getFileStatsIfExists(params.outputFilepath)
|
||||
|
||||
const existing = await InstalledResource.query()
|
||||
.where('resource_id', params.resourceMetadata.resource_id)
|
||||
.where('resource_type', 'map')
|
||||
.first()
|
||||
const oldFilePath = existing?.file_path ?? null
|
||||
|
||||
await InstalledResource.updateOrCreate(
|
||||
{
|
||||
resource_id: params.resourceMetadata.resource_id,
|
||||
resource_type: 'map',
|
||||
},
|
||||
{
|
||||
version: params.resourceMetadata.version,
|
||||
collection_ref: params.resourceMetadata.collection_ref,
|
||||
url: params.sourceUrl,
|
||||
file_path: params.outputFilepath,
|
||||
file_size_bytes: fileStat ? Number(fileStat.size) : null,
|
||||
installed_at: DateTime.now(),
|
||||
}
|
||||
)
|
||||
|
||||
if (oldFilePath && oldFilePath !== params.outputFilepath) {
|
||||
try {
|
||||
await fsUtils.deleteFileIfExists(oldFilePath)
|
||||
} catch (err) {
|
||||
logger.warn(`[RunExtractPmtilesJob] Failed to delete old file ${oldFilePath}: ${err}`)
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: scan the pmtiles dir for orphans with the same resource_id that the DB
|
||||
// lookup above didn't catch — e.g. a prior extract crashed before writing its
|
||||
// InstalledResource row, or an earlier bug wrote a file without registering it.
|
||||
// Matches both curated (`<id>_YYYY-MM.pmtiles`) and regional (`<id>_YYYYMMDD_zN.pmtiles`)
|
||||
// naming — prefix-only so new filename formats don't silently miss.
|
||||
const dir = dirname(params.outputFilepath)
|
||||
const keepName = basename(params.outputFilepath)
|
||||
const prefix = `${params.resourceMetadata.resource_id}_`
|
||||
try {
|
||||
const entries = await readdir(dir)
|
||||
for (const entry of entries) {
|
||||
if (entry === keepName || !entry.endsWith('.pmtiles')) continue
|
||||
if (!entry.startsWith(prefix)) continue
|
||||
const orphanPath = join(dir, entry)
|
||||
if (orphanPath === oldFilePath) continue
|
||||
try {
|
||||
await fsUtils.deleteFileIfExists(orphanPath)
|
||||
logger.info(`[RunExtractPmtilesJob] Pruned orphan pmtiles ${orphanPath}`)
|
||||
} catch (err) {
|
||||
logger.warn(`[RunExtractPmtilesJob] Failed to prune orphan ${orphanPath}: ${err}`)
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(`[RunExtractPmtilesJob] Directory scan for orphans failed: ${err}`)
|
||||
}
|
||||
}
|
||||
|
||||
static async getById(jobId: string): Promise<Job | undefined> {
|
||||
const queueService = new QueueService()
|
||||
const queue = queueService.getQueue(this.queue)
|
||||
return await queue.getJob(jobId)
|
||||
}
|
||||
|
||||
static async dispatch(params: RunExtractPmtilesJobParams) {
|
||||
const queueService = new QueueService()
|
||||
const queue = queueService.getQueue(this.queue)
|
||||
const jobId = this.getJobId(params.sourceUrl, params.regionFilepath, params.maxzoom)
|
||||
|
||||
const existing = await queue.getJob(jobId)
|
||||
if (existing) {
|
||||
const state = await existing.getState()
|
||||
if (state === 'active' || state === 'waiting' || state === 'delayed') {
|
||||
return {
|
||||
job: existing,
|
||||
created: false,
|
||||
message: `Extract job already exists for these params`,
|
||||
}
|
||||
}
|
||||
// Stale (completed/failed) — remove so we can re-dispatch under the same deterministic id
|
||||
try {
|
||||
await existing.remove()
|
||||
} catch {
|
||||
// Already gone or locked — add() below will still report a meaningful error
|
||||
}
|
||||
}
|
||||
|
||||
// Fewer attempts than HTTP downloads — a failed extract usually means the source URL
|
||||
// rotated or the CDN is throttling, and resuming mid-extract isn't supported by the CLI
|
||||
const job = await queue.add(this.key, params, {
|
||||
jobId,
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential', delay: 60000 },
|
||||
removeOnComplete: true,
|
||||
})
|
||||
return {
|
||||
job,
|
||||
created: true,
|
||||
message: `Dispatched pmtiles extract job`,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
import { DateTime } from 'luxon'
|
||||
import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
|
||||
|
||||
export default class CustomLibrarySource extends BaseModel {
|
||||
static namingStrategy = new SnakeCaseNamingStrategy()
|
||||
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
|
||||
@column()
|
||||
declare name: string
|
||||
|
||||
@column()
|
||||
declare base_url: string
|
||||
|
||||
@column()
|
||||
declare is_default: boolean
|
||||
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare created_at: DateTime
|
||||
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updated_at: DateTime
|
||||
}
|
||||
|
|
@ -53,8 +53,10 @@ export class CollectionUpdateService {
|
|||
`[CollectionUpdateService] Update check complete: ${response.data.length} update(s) available`
|
||||
)
|
||||
|
||||
const updates = await this.enrichWithSizes(response.data)
|
||||
|
||||
return {
|
||||
updates: response.data,
|
||||
updates,
|
||||
checked_at: new Date().toISOString(),
|
||||
}
|
||||
} catch (error) {
|
||||
|
|
@ -105,6 +107,8 @@ export class CollectionUpdateService {
|
|||
update.resource_type === 'zim' ? ZIM_MIME_TYPES : PMTILES_MIME_TYPES,
|
||||
forceNew: true,
|
||||
filetype: update.resource_type,
|
||||
title: update.resource_id,
|
||||
totalBytes: update.size_bytes,
|
||||
resourceMetadata: {
|
||||
resource_id: update.resource_id,
|
||||
version: update.latest_version,
|
||||
|
|
@ -126,21 +130,42 @@ export class CollectionUpdateService {
|
|||
async applyAllUpdates(
|
||||
updates: ResourceUpdateInfo[]
|
||||
): Promise<{ results: Array<{ resource_id: string; success: boolean; jobId?: string; error?: string }> }> {
|
||||
const results: Array<{
|
||||
resource_id: string
|
||||
success: boolean
|
||||
jobId?: string
|
||||
error?: string
|
||||
}> = []
|
||||
|
||||
for (const update of updates) {
|
||||
const result = await this.applyUpdate(update)
|
||||
results.push({ resource_id: update.resource_id, ...result })
|
||||
}
|
||||
const results = await Promise.all(
|
||||
updates.map(async (update) => {
|
||||
const result = await this.applyUpdate(update)
|
||||
return { resource_id: update.resource_id, ...result }
|
||||
})
|
||||
)
|
||||
|
||||
return { results }
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch Content-Length for each update URL in parallel. HEAD failures are non-fatal —
|
||||
* the update row just renders without a size. Bounded to HEAD_TIMEOUT_MS so a slow
|
||||
* mirror doesn't block the whole check.
|
||||
*/
|
||||
private async enrichWithSizes(updates: ResourceUpdateInfo[]): Promise<ResourceUpdateInfo[]> {
|
||||
const HEAD_TIMEOUT_MS = 5000
|
||||
|
||||
return await Promise.all(
|
||||
updates.map(async (update) => {
|
||||
if (update.size_bytes) return update // Trust upstream if it already gave us one
|
||||
try {
|
||||
const head = await axios.head(update.download_url, {
|
||||
timeout: HEAD_TIMEOUT_MS,
|
||||
maxRedirects: 5,
|
||||
validateStatus: (s) => s >= 200 && s < 400,
|
||||
})
|
||||
const len = Number(head.headers['content-length'])
|
||||
return Number.isFinite(len) && len > 0 ? { ...update, size_bytes: len } : update
|
||||
} catch {
|
||||
return update
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
private buildFilename(update: ResourceUpdateInfo): string {
|
||||
if (update.resource_type === 'zim') {
|
||||
return `${update.resource_id}_${update.latest_version}.zim`
|
||||
|
|
|
|||
|
|
@ -0,0 +1,308 @@
|
|||
import { access, readFile, writeFile, mkdir } from 'fs/promises'
|
||||
import { join, resolve } from 'path'
|
||||
import { createHash } from 'crypto'
|
||||
import { tmpdir } from 'os'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import type { Country, CountryCode, CountryGroup } from '../../types/maps.js'
|
||||
|
||||
interface NEFeature {
|
||||
type: 'Feature'
|
||||
properties: Record<string, any>
|
||||
geometry: unknown
|
||||
}
|
||||
|
||||
interface NEFeatureCollection {
|
||||
type: 'FeatureCollection'
|
||||
features: NEFeature[]
|
||||
}
|
||||
|
||||
const COUNTRY_GEOJSON_PATH = join(
|
||||
process.cwd(),
|
||||
'resources',
|
||||
'geodata',
|
||||
'ne_50m_admin_0_countries.geojson'
|
||||
)
|
||||
|
||||
// Natural Earth country polygons are land-only (no territorial waters), so a
|
||||
// strict intersect leaves tiles fully over the ocean out of the extract —
|
||||
// coastal cities render as grey off their coast. Inflate each polygon outward
|
||||
// by ~11 km to pull in adjacent tiles without ballooning the extract size.
|
||||
const REGION_BUFFER_DEGREES = 0.1
|
||||
|
||||
const GROUP_ORDER = [
|
||||
'north-america',
|
||||
'south-america',
|
||||
'europe',
|
||||
'africa',
|
||||
'asia',
|
||||
'oceania',
|
||||
]
|
||||
|
||||
const GROUP_META: Record<string, { id: string; name: string; description: string }> = {
|
||||
'North America': {
|
||||
id: 'north-america',
|
||||
name: 'North America',
|
||||
description: 'All countries in North America and the Caribbean.',
|
||||
},
|
||||
'South America': {
|
||||
id: 'south-america',
|
||||
name: 'South America',
|
||||
description: 'All countries in South America.',
|
||||
},
|
||||
Europe: {
|
||||
id: 'europe',
|
||||
name: 'Europe',
|
||||
description: 'All countries in Europe.',
|
||||
},
|
||||
Africa: {
|
||||
id: 'africa',
|
||||
name: 'Africa',
|
||||
description: 'All countries in Africa.',
|
||||
},
|
||||
Asia: {
|
||||
id: 'asia',
|
||||
name: 'Asia',
|
||||
description: 'All countries in Asia.',
|
||||
},
|
||||
Oceania: {
|
||||
id: 'oceania',
|
||||
name: 'Oceania',
|
||||
description: 'Australia, New Zealand, and Pacific island nations.',
|
||||
},
|
||||
}
|
||||
|
||||
export class CountriesService {
|
||||
private static instance: CountriesService | null = null
|
||||
private loadPromise: Promise<void> | null = null
|
||||
private countries: Country[] = []
|
||||
private byCode: Map<CountryCode, { country: Country; feature: NEFeature }> = new Map()
|
||||
private groups: CountryGroup[] = []
|
||||
|
||||
static getInstance(): CountriesService {
|
||||
if (!this.instance) {
|
||||
this.instance = new CountriesService()
|
||||
}
|
||||
return this.instance
|
||||
}
|
||||
|
||||
private async ensureLoaded(): Promise<void> {
|
||||
if (this.byCode.size > 0) return
|
||||
if (!this.loadPromise) {
|
||||
this.loadPromise = this.load()
|
||||
}
|
||||
await this.loadPromise
|
||||
}
|
||||
|
||||
private async load(): Promise<void> {
|
||||
const raw = await readFile(COUNTRY_GEOJSON_PATH, 'utf8')
|
||||
const fc = JSON.parse(raw) as NEFeatureCollection
|
||||
|
||||
// Natural Earth reuses a sovereign state's ISO_A2 for its dependencies
|
||||
// (e.g. AU covers both Australia and Australian territories). Sort so the
|
||||
// sovereign mainland wins the ISO-code slot, and skip any subsequent
|
||||
// same-code dependency — otherwise the "AU" entry ends up being some tiny
|
||||
// island territory.
|
||||
const sortedFeatures = [...fc.features].sort((a, b) => typeRank(a) - typeRank(b))
|
||||
|
||||
const countries: Country[] = []
|
||||
const byCode = new Map<CountryCode, { country: Country; feature: NEFeature }>()
|
||||
const groupCodes: Record<string, CountryCode[]> = {}
|
||||
|
||||
for (const feature of sortedFeatures) {
|
||||
const p = feature.properties
|
||||
const code = resolveIso2(p)
|
||||
if (!code) continue
|
||||
if (byCode.has(code)) continue
|
||||
|
||||
const continent = typeof p.CONTINENT === 'string' ? p.CONTINENT : 'Other'
|
||||
if (continent === 'Antarctica' || continent === 'Seven seas (open ocean)') continue
|
||||
|
||||
const country: Country = {
|
||||
code,
|
||||
code3: resolveIso3(p) ?? code,
|
||||
name: typeof p.NAME === 'string' ? p.NAME : code,
|
||||
continent,
|
||||
subregion: typeof p.SUBREGION === 'string' ? p.SUBREGION : continent,
|
||||
population: typeof p.POP_EST === 'number' ? p.POP_EST : 0,
|
||||
}
|
||||
|
||||
countries.push(country)
|
||||
byCode.set(code, { country, feature })
|
||||
|
||||
if (GROUP_META[continent]) {
|
||||
const groupId = GROUP_META[continent].id
|
||||
if (!groupCodes[groupId]) groupCodes[groupId] = []
|
||||
groupCodes[groupId].push(code)
|
||||
}
|
||||
}
|
||||
|
||||
countries.sort((a, b) => a.name.localeCompare(b.name))
|
||||
|
||||
const groups: CountryGroup[] = GROUP_ORDER.flatMap((groupId) => {
|
||||
const meta = Object.values(GROUP_META).find((m) => m.id === groupId)
|
||||
if (!meta) return []
|
||||
const codes = (groupCodes[groupId] ?? []).slice().sort()
|
||||
if (codes.length === 0) return []
|
||||
return [{ id: meta.id, name: meta.name, description: meta.description, countries: codes }]
|
||||
})
|
||||
|
||||
this.countries = countries
|
||||
this.byCode = byCode
|
||||
this.groups = groups
|
||||
|
||||
logger.info(
|
||||
`[CountriesService] Loaded ${countries.length} countries across ${groups.length} groups`
|
||||
)
|
||||
}
|
||||
|
||||
async list(): Promise<Country[]> {
|
||||
await this.ensureLoaded()
|
||||
return this.countries
|
||||
}
|
||||
|
||||
async listGroups(): Promise<CountryGroup[]> {
|
||||
await this.ensureLoaded()
|
||||
return this.groups
|
||||
}
|
||||
|
||||
/** Throws when a supplied code does not map to a known country. */
|
||||
async resolveCodes(codes: CountryCode[]): Promise<CountryCode[]> {
|
||||
await this.ensureLoaded()
|
||||
const normalized = [...new Set(codes.map((c) => c.toUpperCase()))].sort()
|
||||
const unknown = normalized.filter((c) => !this.byCode.has(c))
|
||||
if (unknown.length > 0) {
|
||||
throw new Error(`Unknown country code(s): ${unknown.join(', ')}`)
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
/**
|
||||
* Filename is keyed on a hash of the sorted ISO codes + buffer size so
|
||||
* repeated calls with the same selection reuse the same path, and bumping
|
||||
* the buffer auto-invalidates stale files.
|
||||
*/
|
||||
async writeRegionFile(codes: CountryCode[]): Promise<string> {
|
||||
await this.ensureLoaded()
|
||||
const resolved = await this.resolveCodes(codes)
|
||||
const key = `b${REGION_BUFFER_DEGREES}:${resolved.join(',')}`
|
||||
const hash = createHash('sha1').update(key).digest('hex').slice(0, 12)
|
||||
|
||||
const dir = resolve(tmpdir(), 'nomad-pmtiles-regions')
|
||||
await mkdir(dir, { recursive: true })
|
||||
const filepath = join(dir, `region-${hash}.geojson`)
|
||||
|
||||
try {
|
||||
await access(filepath)
|
||||
return filepath
|
||||
} catch {}
|
||||
|
||||
const fc = {
|
||||
type: 'FeatureCollection',
|
||||
features: resolved.map((code) => {
|
||||
const entry = this.byCode.get(code)!
|
||||
return {
|
||||
type: 'Feature',
|
||||
properties: { iso: code, name: entry.country.name },
|
||||
geometry: bufferGeometry(entry.feature.geometry, REGION_BUFFER_DEGREES),
|
||||
}
|
||||
}),
|
||||
}
|
||||
|
||||
await writeFile(filepath, JSON.stringify(fc))
|
||||
return filepath
|
||||
}
|
||||
}
|
||||
|
||||
function typeRank(f: NEFeature): number {
|
||||
const t = typeof f.properties.TYPE === 'string' ? f.properties.TYPE : ''
|
||||
if (t === 'Sovereign country') return 0
|
||||
if (t === 'Country') return 1
|
||||
if (t === 'Sovereignty') return 2
|
||||
if (t === 'Disputed') return 3
|
||||
if (t === 'Dependency') return 4
|
||||
return 5
|
||||
}
|
||||
|
||||
function resolveIso2(p: Record<string, any>): CountryCode | null {
|
||||
// Natural Earth's ISO_A2 sometimes holds political escapes like "CN-TW" for
|
||||
// Taiwan or "-99" for countries involved in disputes. Only accept clean
|
||||
// 2-letter codes; fall back to ISO_A2_EH (which reliably has the real code).
|
||||
const primary = typeof p.ISO_A2 === 'string' ? p.ISO_A2 : null
|
||||
if (primary && /^[A-Z]{2}$/i.test(primary)) return primary.toUpperCase()
|
||||
const fallback = typeof p.ISO_A2_EH === 'string' ? p.ISO_A2_EH : null
|
||||
if (fallback && /^[A-Z]{2}$/i.test(fallback)) return fallback.toUpperCase()
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Inflate each polygon ring outward by `buffer` degrees via per-vertex
|
||||
* averaged-normal offset. Not geodesically accurate — but at small buffers
|
||||
* (<= 0.2°) it's within a few percent of a proper geodesic buffer at
|
||||
* country scale, which is plenty for tile-inclusion purposes.
|
||||
*/
|
||||
function bufferGeometry(geometry: unknown, buffer: number): unknown {
|
||||
const geom = geometry as { type: string; coordinates: any }
|
||||
if (geom?.type === 'Polygon') {
|
||||
return { type: 'Polygon', coordinates: bufferPolygonRings(geom.coordinates, buffer) }
|
||||
}
|
||||
if (geom?.type === 'MultiPolygon') {
|
||||
return {
|
||||
type: 'MultiPolygon',
|
||||
coordinates: geom.coordinates.map((poly: number[][][]) =>
|
||||
bufferPolygonRings(poly, buffer)
|
||||
),
|
||||
}
|
||||
}
|
||||
return geometry
|
||||
}
|
||||
|
||||
function bufferPolygonRings(rings: number[][][], buffer: number): number[][][] {
|
||||
return rings.map((ring) => bufferRing(ring, buffer))
|
||||
}
|
||||
|
||||
function bufferRing(ring: number[][], buffer: number): number[][] {
|
||||
if (ring.length < 4) return ring
|
||||
const sign = signedArea(ring) > 0 ? 1 : -1
|
||||
const n = ring.length - 1
|
||||
const out: number[][] = []
|
||||
for (let i = 0; i < n; i++) {
|
||||
const prev = ring[(i - 1 + n) % n]
|
||||
const curr = ring[i]
|
||||
const next = ring[(i + 1) % n]
|
||||
const e1x = curr[0] - prev[0]
|
||||
const e1y = curr[1] - prev[1]
|
||||
const e2x = next[0] - curr[0]
|
||||
const e2y = next[1] - curr[1]
|
||||
const l1 = Math.hypot(e1x, e1y) || 1
|
||||
const l2 = Math.hypot(e2x, e2y) || 1
|
||||
const n1x = (e1y / l1) * sign
|
||||
const n1y = (-e1x / l1) * sign
|
||||
const n2x = (e2y / l2) * sign
|
||||
const n2y = (-e2x / l2) * sign
|
||||
const sumX = n1x + n2x
|
||||
const sumY = n1y + n2y
|
||||
const sl = Math.hypot(sumX, sumY) || 1
|
||||
out.push([curr[0] + (sumX / sl) * buffer, curr[1] + (sumY / sl) * buffer])
|
||||
}
|
||||
out.push(out[0])
|
||||
return out
|
||||
}
|
||||
|
||||
function signedArea(ring: number[][]): number {
|
||||
let a = 0
|
||||
for (let i = 0; i < ring.length - 1; i++) {
|
||||
a += ring[i][0] * ring[i + 1][1] - ring[i + 1][0] * ring[i][1]
|
||||
}
|
||||
return a / 2
|
||||
}
|
||||
|
||||
function resolveIso3(p: Record<string, any>): string | null {
|
||||
const primary = typeof p.ISO_A3 === 'string' ? p.ISO_A3 : null
|
||||
if (primary && primary !== '-99') return primary.toUpperCase()
|
||||
const fallback = typeof p.ISO_A3_EH === 'string' ? p.ISO_A3_EH : null
|
||||
if (fallback && fallback !== '-99') return fallback.toUpperCase()
|
||||
const adm = typeof p.ADM0_A3 === 'string' ? p.ADM0_A3 : null
|
||||
if (adm && adm !== '-99') return adm.toUpperCase()
|
||||
return null
|
||||
}
|
||||
|
||||
|
|
@ -10,7 +10,7 @@ import { KiwixLibraryService } from './kiwix_library_service.js'
|
|||
import { SERVICE_NAMES } from '../../constants/service_names.js'
|
||||
import { exec } from 'child_process'
|
||||
import { promisify } from 'util'
|
||||
// import { readdir } from 'fs/promises'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import KVStore from '#models/kv_store'
|
||||
import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
|
||||
import { KIWIX_LIBRARY_CMD } from '../../constants/kiwix.js'
|
||||
|
|
@ -500,6 +500,7 @@ export class DockerService {
|
|||
// GPU-aware configuration for Ollama
|
||||
let finalImage = service.container_image
|
||||
let gpuHostConfig = containerConfig?.HostConfig || {}
|
||||
let amdGpuConfigured = false
|
||||
|
||||
if (service.service_name === SERVICE_NAMES.OLLAMA) {
|
||||
const gpuResult = await this._detectGPUType()
|
||||
|
|
@ -523,16 +524,51 @@ export class DockerService {
|
|||
],
|
||||
}
|
||||
} else if (gpuResult.type === 'amd') {
|
||||
this._broadcast(
|
||||
service.service_name,
|
||||
'gpu-config',
|
||||
`AMD GPU detected. ROCm GPU acceleration is not yet supported in this version — proceeding with CPU-only configuration. GPU support for AMD will be available in a future update.`
|
||||
)
|
||||
logger.warn('[DockerService] AMD GPU detected but ROCm support is not yet enabled. Using CPU-only configuration.')
|
||||
// TODO: Re-enable AMD GPU support once ROCm image and device discovery are validated.
|
||||
// When re-enabling:
|
||||
// 1. Switch image to 'ollama/ollama:rocm'
|
||||
// 2. Restore _discoverAMDDevices() to map /dev/kfd and /dev/dri/* into the container
|
||||
// AMD acceleration is opt-out via the 'ai.amdGpuAcceleration' KV key (default-on).
|
||||
// Per memory feedback: KV values can be string or boolean — coerce explicitly.
|
||||
const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration')
|
||||
const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false'
|
||||
|
||||
if (amdAccelerationEnabled) {
|
||||
this._broadcast(
|
||||
service.service_name,
|
||||
'gpu-config',
|
||||
`AMD GPU detected. Using ROCm image with /dev/kfd and /dev/dri passthrough...`
|
||||
)
|
||||
|
||||
finalImage = 'ollama/ollama:rocm'
|
||||
|
||||
// The pull-if-missing earlier in this function used service.container_image
|
||||
// (the DB-pinned tag, e.g. ollama/ollama:0.18.2). The AMD branch overrides
|
||||
// to a different tag — so we need to pull :rocm separately if it's not local.
|
||||
const rocmImageExists = await this._checkImageExists(finalImage)
|
||||
if (!rocmImageExists) {
|
||||
this._broadcast(
|
||||
service.service_name,
|
||||
'pulling',
|
||||
`Pulling Docker image ${finalImage}...`
|
||||
)
|
||||
const rocmPullStream = await this.docker.pull(finalImage)
|
||||
await new Promise((res) => this.docker.modem.followProgress(rocmPullStream, res))
|
||||
}
|
||||
|
||||
const amdDevices = await this._discoverAMDDevices()
|
||||
gpuHostConfig = {
|
||||
...gpuHostConfig,
|
||||
Devices: amdDevices,
|
||||
}
|
||||
amdGpuConfigured = true
|
||||
logger.info(
|
||||
`[DockerService] Configured ROCm image and ${amdDevices.length} AMD device entries for Ollama`
|
||||
)
|
||||
} else {
|
||||
this._broadcast(
|
||||
service.service_name,
|
||||
'gpu-config',
|
||||
`AMD GPU detected but acceleration is disabled via ai.amdGpuAcceleration. Using CPU-only configuration.`
|
||||
)
|
||||
logger.info('[DockerService] AMD GPU acceleration disabled by KV opt-out; using CPU-only configuration.')
|
||||
}
|
||||
} else if (gpuResult.toolkitMissing) {
|
||||
this._broadcast(
|
||||
service.service_name,
|
||||
|
|
@ -555,6 +591,12 @@ export class DockerService {
|
|||
if (flashAttentionEnabled !== false) {
|
||||
ollamaEnv.push('OLLAMA_FLASH_ATTENTION=1')
|
||||
}
|
||||
if (amdGpuConfigured) {
|
||||
// RDNA3 iGPUs (gfx1103: 780M, 880M, 890M, ...) aren't on AMD's official ROCm
|
||||
// allowlist but work when forced to identify as gfx1100 via HSA_OVERRIDE_GFX_VERSION.
|
||||
// Harmless on supported discrete cards (gfx1030 RX 6800, etc.) — they ignore the override.
|
||||
ollamaEnv.push('HSA_OVERRIDE_GFX_VERSION=11.0.0')
|
||||
}
|
||||
}
|
||||
|
||||
this._broadcast(
|
||||
|
|
@ -857,7 +899,10 @@ export class DockerService {
|
|||
/**
|
||||
* Detect GPU type and toolkit availability.
|
||||
* Primary: Check Docker runtimes via docker.info() (works from inside containers).
|
||||
* Fallback: lspci for host-based installs and AMD detection.
|
||||
* Secondary: Read /app/storage/.nomad-gpu-type written by install_nomad.sh — needed
|
||||
* for AMD detection because lspci isn't available inside the admin container and
|
||||
* AMD has no Docker runtime registration to query.
|
||||
* Fallback: lspci for host-based installs.
|
||||
*/
|
||||
private async _detectGPUType(): Promise<{ type: 'nvidia' | 'amd' | 'none'; toolkitMissing?: boolean }> {
|
||||
try {
|
||||
|
|
@ -874,6 +919,24 @@ export class DockerService {
|
|||
logger.warn(`[DockerService] Could not query Docker info for GPU runtimes: ${error.message}`)
|
||||
}
|
||||
|
||||
// Secondary: install_nomad.sh writes the host-detected GPU type to a marker file in
|
||||
// the storage volume so the admin container (which lacks lspci) can read it.
|
||||
try {
|
||||
const marker = (await readFile('/app/storage/.nomad-gpu-type', 'utf8')).trim()
|
||||
if (marker === 'nvidia') {
|
||||
// Hardware present but Docker doesn't have nvidia runtime → toolkit missing
|
||||
logger.warn('[DockerService] NVIDIA GPU recorded in marker file but NVIDIA Container Toolkit is not installed')
|
||||
return { type: 'none', toolkitMissing: true }
|
||||
}
|
||||
if (marker === 'amd') {
|
||||
logger.info('[DockerService] AMD GPU detected via install-time marker file')
|
||||
await this._persistGPUType('amd')
|
||||
return { type: 'amd' }
|
||||
}
|
||||
} catch {
|
||||
// No marker file — fall through to lspci attempt for host-based installs
|
||||
}
|
||||
|
||||
// Fallback: lspci for host-based installs (not available inside Docker)
|
||||
const execAsync = promisify(exec)
|
||||
|
||||
|
|
@ -937,60 +1000,23 @@ export class DockerService {
|
|||
}
|
||||
|
||||
/**
|
||||
* Discover AMD GPU DRI devices dynamically.
|
||||
* Returns an array of device configurations for Docker.
|
||||
* Build the Docker Devices array for AMD GPU passthrough.
|
||||
*
|
||||
* Returns /dev/kfd (Kernel Fusion Driver, required by ROCm) and /dev/dri (the DRM
|
||||
* device tree). Passing /dev/dri as a single directory entry mirrors Docker CLI
|
||||
* --device behavior — the daemon expands it to all child devices (card*, renderD*)
|
||||
* regardless of how the host enumerates them. This avoids the brittle hardcoded
|
||||
* fallback (card0/renderD128) the prior implementation used, which was wrong on
|
||||
* systems where the AMD GPU enumerates as card1+ (e.g. UM890 Pro 780M iGPU).
|
||||
*/
|
||||
// private async _discoverAMDDevices(): Promise<
|
||||
// Array<{ PathOnHost: string; PathInContainer: string; CgroupPermissions: string }>
|
||||
// > {
|
||||
// try {
|
||||
// const devices: Array<{
|
||||
// PathOnHost: string
|
||||
// PathInContainer: string
|
||||
// CgroupPermissions: string
|
||||
// }> = []
|
||||
|
||||
// // Always add /dev/kfd (Kernel Fusion Driver)
|
||||
// devices.push({
|
||||
// PathOnHost: '/dev/kfd',
|
||||
// PathInContainer: '/dev/kfd',
|
||||
// CgroupPermissions: 'rwm',
|
||||
// })
|
||||
|
||||
// // Discover DRI devices in /dev/dri/
|
||||
// try {
|
||||
// const driDevices = await readdir('/dev/dri')
|
||||
// for (const device of driDevices) {
|
||||
// const devicePath = `/dev/dri/${device}`
|
||||
// devices.push({
|
||||
// PathOnHost: devicePath,
|
||||
// PathInContainer: devicePath,
|
||||
// CgroupPermissions: 'rwm',
|
||||
// })
|
||||
// }
|
||||
// logger.info(
|
||||
// `[DockerService] Discovered ${driDevices.length} DRI devices: ${driDevices.join(', ')}`
|
||||
// )
|
||||
// } catch (error) {
|
||||
// logger.warn(`[DockerService] Could not read /dev/dri directory: ${error.message}`)
|
||||
// // Fallback to common device names if directory read fails
|
||||
// const fallbackDevices = ['card0', 'renderD128']
|
||||
// for (const device of fallbackDevices) {
|
||||
// devices.push({
|
||||
// PathOnHost: `/dev/dri/${device}`,
|
||||
// PathInContainer: `/dev/dri/${device}`,
|
||||
// CgroupPermissions: 'rwm',
|
||||
// })
|
||||
// }
|
||||
// logger.info(`[DockerService] Using fallback DRI devices: ${fallbackDevices.join(', ')}`)
|
||||
// }
|
||||
|
||||
// return devices
|
||||
// } catch (error) {
|
||||
// logger.error(`[DockerService] Error discovering AMD devices: ${error.message}`)
|
||||
// return []
|
||||
// }
|
||||
// }
|
||||
private async _discoverAMDDevices(): Promise<
|
||||
Array<{ PathOnHost: string; PathInContainer: string; CgroupPermissions: string }>
|
||||
> {
|
||||
return [
|
||||
{ PathOnHost: '/dev/kfd', PathInContainer: '/dev/kfd', CgroupPermissions: 'rwm' },
|
||||
{ PathOnHost: '/dev/dri', PathInContainer: '/dev/dri', CgroupPermissions: 'rwm' },
|
||||
]
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a service container to a new image version while preserving volumes and data.
|
||||
|
|
@ -1014,12 +1040,60 @@ export class DockerService {
|
|||
|
||||
this.activeInstallations.add(serviceName)
|
||||
|
||||
// Compute new image string
|
||||
// Compute new image string. AMD-on-Ollama overrides this to the rolling :rocm tag
|
||||
// (set during GPU detection below) since per-version ROCm tags aren't always published.
|
||||
const currentImage = service.container_image
|
||||
const imageBase = currentImage.includes(':')
|
||||
? currentImage.substring(0, currentImage.lastIndexOf(':'))
|
||||
: currentImage
|
||||
const newImage = `${imageBase}:${targetVersion}`
|
||||
let newImage = `${imageBase}:${targetVersion}`
|
||||
|
||||
// GPU detection runs before the pull so AMD updates pull ollama/ollama:rocm rather
|
||||
// than the standard tag. Detection result is reused below when building the new
|
||||
// container config (devices, env). Non-Ollama services skip this entirely.
|
||||
let updatedDeviceRequests: any[] | undefined = undefined
|
||||
let updatedAmdDevices: any[] | undefined = undefined
|
||||
let updatedAmdGpuConfigured = false
|
||||
if (serviceName === SERVICE_NAMES.OLLAMA) {
|
||||
const gpuResult = await this._detectGPUType()
|
||||
if (gpuResult.type === 'nvidia') {
|
||||
this._broadcast(
|
||||
serviceName,
|
||||
'update-gpu-config',
|
||||
`NVIDIA container runtime detected. Configuring updated container with GPU support...`
|
||||
)
|
||||
updatedDeviceRequests = [
|
||||
{ Driver: 'nvidia', Count: -1, Capabilities: [['gpu']] },
|
||||
]
|
||||
} else if (gpuResult.type === 'amd') {
|
||||
const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration')
|
||||
const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false'
|
||||
if (amdAccelerationEnabled) {
|
||||
this._broadcast(
|
||||
serviceName,
|
||||
'update-gpu-config',
|
||||
`AMD GPU detected. Using ROCm image with /dev/kfd and /dev/dri passthrough...`
|
||||
)
|
||||
newImage = 'ollama/ollama:rocm'
|
||||
updatedAmdDevices = await this._discoverAMDDevices()
|
||||
updatedAmdGpuConfigured = true
|
||||
} else {
|
||||
this._broadcast(
|
||||
serviceName,
|
||||
'update-gpu-config',
|
||||
`AMD GPU detected but acceleration is disabled via ai.amdGpuAcceleration. Using CPU-only configuration.`
|
||||
)
|
||||
}
|
||||
} else if (gpuResult.toolkitMissing) {
|
||||
this._broadcast(
|
||||
serviceName,
|
||||
'update-gpu-config',
|
||||
`NVIDIA GPU detected but NVIDIA Container Toolkit is not installed. Using CPU-only configuration. Install the toolkit and reinstall AI Assistant for GPU acceleration: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html`
|
||||
)
|
||||
} else {
|
||||
this._broadcast(serviceName, 'update-gpu-config', `No GPU detected. Using CPU-only configuration.`)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 1: Pull new image
|
||||
this._broadcast(serviceName, 'update-pulling', `Pulling image ${newImage}...`)
|
||||
|
|
@ -1054,48 +1128,21 @@ export class DockerService {
|
|||
|
||||
const hostConfig = inspectData.HostConfig || {}
|
||||
|
||||
// Re-run GPU detection for Ollama so updates always reflect the current GPU environment.
|
||||
// This handles cases where the NVIDIA Container Toolkit was installed after the initial
|
||||
// Ollama setup, and ensures DeviceRequests are always built fresh rather than relying on
|
||||
// round-tripping the Docker inspect format back into the create API.
|
||||
let updatedDeviceRequests: any[] | undefined = undefined
|
||||
if (serviceName === SERVICE_NAMES.OLLAMA) {
|
||||
const gpuResult = await this._detectGPUType()
|
||||
|
||||
if (gpuResult.type === 'nvidia') {
|
||||
this._broadcast(
|
||||
serviceName,
|
||||
'update-gpu-config',
|
||||
`NVIDIA container runtime detected. Configuring updated container with GPU support...`
|
||||
)
|
||||
updatedDeviceRequests = [
|
||||
{
|
||||
Driver: 'nvidia',
|
||||
Count: -1,
|
||||
Capabilities: [['gpu']],
|
||||
},
|
||||
// GPU detection already ran above (before the pull) so we know the right image, devices,
|
||||
// and whether HSA_OVERRIDE needs injection. For AMD, replace any prior HSA_OVERRIDE in
|
||||
// the inspect-captured env so updates from older containers pick up the current value.
|
||||
const baseEnv = inspectData.Config?.Env || []
|
||||
const finalEnv = updatedAmdGpuConfigured
|
||||
? [
|
||||
...baseEnv.filter((e: string) => !e.startsWith('HSA_OVERRIDE_GFX_VERSION=')),
|
||||
'HSA_OVERRIDE_GFX_VERSION=11.0.0',
|
||||
]
|
||||
} else if (gpuResult.type === 'amd') {
|
||||
this._broadcast(
|
||||
serviceName,
|
||||
'update-gpu-config',
|
||||
`AMD GPU detected. ROCm GPU acceleration is not yet supported — using CPU-only configuration.`
|
||||
)
|
||||
} else if (gpuResult.toolkitMissing) {
|
||||
this._broadcast(
|
||||
serviceName,
|
||||
'update-gpu-config',
|
||||
`NVIDIA GPU detected but NVIDIA Container Toolkit is not installed. Using CPU-only configuration. Install the toolkit and reinstall AI Assistant for GPU acceleration: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html`
|
||||
)
|
||||
} else {
|
||||
this._broadcast(serviceName, 'update-gpu-config', `No GPU detected. Using CPU-only configuration.`)
|
||||
}
|
||||
}
|
||||
: baseEnv
|
||||
|
||||
const newContainerConfig: any = {
|
||||
Image: newImage,
|
||||
name: serviceName,
|
||||
Env: inspectData.Config?.Env || undefined,
|
||||
Env: finalEnv.length > 0 ? finalEnv : undefined,
|
||||
Cmd: inspectData.Config?.Cmd || undefined,
|
||||
ExposedPorts: inspectData.Config?.ExposedPorts || undefined,
|
||||
WorkingDir: inspectData.Config?.WorkingDir || undefined,
|
||||
|
|
@ -1105,7 +1152,7 @@ export class DockerService {
|
|||
PortBindings: hostConfig.PortBindings || undefined,
|
||||
RestartPolicy: hostConfig.RestartPolicy || undefined,
|
||||
DeviceRequests: serviceName === SERVICE_NAMES.OLLAMA ? updatedDeviceRequests : (hostConfig.DeviceRequests || undefined),
|
||||
Devices: hostConfig.Devices || undefined,
|
||||
Devices: serviceName === SERVICE_NAMES.OLLAMA && updatedAmdDevices ? updatedAmdDevices : (hostConfig.Devices || undefined),
|
||||
},
|
||||
NetworkingConfig: inspectData.NetworkSettings?.Networks
|
||||
? {
|
||||
|
|
|
|||
|
|
@ -1,13 +1,19 @@
|
|||
import { inject } from '@adonisjs/core'
|
||||
import { QueueService } from './queue_service.js'
|
||||
import { RunDownloadJob } from '#jobs/run_download_job'
|
||||
import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job'
|
||||
import type { RunExtractPmtilesJobParams } from '#jobs/run_extract_pmtiles_job'
|
||||
import { DownloadModelJob } from '#jobs/download_model_job'
|
||||
import { DownloadJobWithProgress, DownloadProgressData } from '../../types/downloads.js'
|
||||
import type { Job, Queue } from 'bullmq'
|
||||
import { normalize } from 'path'
|
||||
import { deleteFileIfExists } from '../utils/fs.js'
|
||||
import transmit from '@adonisjs/transmit/services/main'
|
||||
import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
|
||||
|
||||
type FileJobState = 'waiting' | 'active' | 'delayed' | 'failed'
|
||||
type TaggedJob = { job: Job; state: FileJobState }
|
||||
|
||||
@inject()
|
||||
export class DownloadService {
|
||||
constructor(private queueService: QueueService) {}
|
||||
|
|
@ -26,27 +32,32 @@ export class DownloadService {
|
|||
return { percent: parseInt(String(progress), 10) || 0 }
|
||||
}
|
||||
|
||||
async listDownloadJobs(filetype?: string): Promise<DownloadJobWithProgress[]> {
|
||||
// Get regular file download jobs (zim, map, etc.) — query each state separately so we can
|
||||
// tag each job with its actual BullMQ state rather than guessing from progress data.
|
||||
const queue = this.queueService.getQueue(RunDownloadJob.queue)
|
||||
type FileJobState = 'waiting' | 'active' | 'delayed' | 'failed'
|
||||
|
||||
const [waitingJobs, activeJobs, delayedJobs, failedJobs] = await Promise.all([
|
||||
/** Fetch all non-completed jobs from a queue, tagged with their current BullMQ state */
|
||||
private async fetchJobsWithStates(queueName: string): Promise<TaggedJob[]> {
|
||||
const queue = this.queueService.getQueue(queueName)
|
||||
const [waiting, active, delayed, failed] = await Promise.all([
|
||||
queue.getJobs(['waiting']),
|
||||
queue.getJobs(['active']),
|
||||
queue.getJobs(['delayed']),
|
||||
queue.getJobs(['failed']),
|
||||
])
|
||||
|
||||
const taggedFileJobs: Array<{ job: (typeof waitingJobs)[0]; state: FileJobState }> = [
|
||||
...waitingJobs.map((j) => ({ job: j, state: 'waiting' as const })),
|
||||
...activeJobs.map((j) => ({ job: j, state: 'active' as const })),
|
||||
...delayedJobs.map((j) => ({ job: j, state: 'delayed' as const })),
|
||||
...failedJobs.map((j) => ({ job: j, state: 'failed' as const })),
|
||||
return [
|
||||
...waiting.map((j) => ({ job: j, state: 'waiting' as const })),
|
||||
...active.map((j) => ({ job: j, state: 'active' as const })),
|
||||
...delayed.map((j) => ({ job: j, state: 'delayed' as const })),
|
||||
...failed.map((j) => ({ job: j, state: 'failed' as const })),
|
||||
]
|
||||
}
|
||||
|
||||
const fileDownloads = taggedFileJobs.map(({ job, state }) => {
|
||||
async listDownloadJobs(filetype?: string): Promise<DownloadJobWithProgress[]> {
|
||||
const modelQueue = this.queueService.getQueue(DownloadModelJob.queue)
|
||||
const [fileTagged, extractTagged, modelJobs] = await Promise.all([
|
||||
this.fetchJobsWithStates(RunDownloadJob.queue),
|
||||
this.fetchJobsWithStates(RunExtractPmtilesJob.queue),
|
||||
modelQueue.getJobs(['waiting', 'active', 'delayed', 'failed']),
|
||||
])
|
||||
|
||||
const fileDownloads = fileTagged.map(({ job, state }) => {
|
||||
const parsed = this.parseProgress(job.progress)
|
||||
return {
|
||||
jobId: job.id!.toString(),
|
||||
|
|
@ -63,26 +74,36 @@ export class DownloadService {
|
|||
}
|
||||
})
|
||||
|
||||
// Get Ollama model download jobs
|
||||
const modelQueue = this.queueService.getQueue(DownloadModelJob.queue)
|
||||
const modelJobs = await modelQueue.getJobs(['waiting', 'active', 'delayed', 'failed'])
|
||||
const extractDownloads = extractTagged.map(({ job, state }) => {
|
||||
const parsed = this.parseProgress(job.progress)
|
||||
return {
|
||||
jobId: job.id!.toString(),
|
||||
url: job.data.sourceUrl,
|
||||
progress: parsed.percent,
|
||||
filepath: normalize(job.data.outputFilepath),
|
||||
filetype: job.data.filetype || 'map',
|
||||
title: job.data.title || undefined,
|
||||
downloadedBytes: parsed.downloadedBytes,
|
||||
totalBytes: parsed.totalBytes || job.data.estimatedBytes || undefined,
|
||||
lastProgressTime: parsed.lastProgressTime,
|
||||
status: state,
|
||||
failedReason: job.failedReason || undefined,
|
||||
}
|
||||
})
|
||||
|
||||
const modelDownloads = modelJobs.map((job) => ({
|
||||
jobId: job.id!.toString(),
|
||||
url: job.data.modelName || 'Unknown Model', // Use model name as url
|
||||
url: job.data.modelName || 'Unknown Model',
|
||||
progress: parseInt(job.progress.toString(), 10),
|
||||
filepath: job.data.modelName || 'Unknown Model', // Use model name as filepath
|
||||
filepath: job.data.modelName || 'Unknown Model',
|
||||
filetype: 'model',
|
||||
status: (job.failedReason ? 'failed' : 'active') as 'active' | 'failed',
|
||||
failedReason: job.failedReason || undefined,
|
||||
}))
|
||||
|
||||
const allDownloads = [...fileDownloads, ...modelDownloads]
|
||||
|
||||
// Filter by filetype if specified
|
||||
const allDownloads = [...fileDownloads, ...extractDownloads, ...modelDownloads]
|
||||
const filtered = allDownloads.filter((job) => !filetype || job.filetype === filetype)
|
||||
|
||||
// Sort: active downloads first (by progress desc), then failed at the bottom
|
||||
return filtered.sort((a, b) => {
|
||||
if (a.status === 'failed' && b.status !== 'failed') return 1
|
||||
if (a.status !== 'failed' && b.status === 'failed') return -1
|
||||
|
|
@ -91,7 +112,11 @@ export class DownloadService {
|
|||
}
|
||||
|
||||
async removeFailedJob(jobId: string): Promise<void> {
|
||||
for (const queueName of [RunDownloadJob.queue, DownloadModelJob.queue]) {
|
||||
for (const queueName of [
|
||||
RunDownloadJob.queue,
|
||||
RunExtractPmtilesJob.queue,
|
||||
DownloadModelJob.queue,
|
||||
]) {
|
||||
const queue = this.queueService.getQueue(queueName)
|
||||
const job = await queue.getJob(jobId)
|
||||
if (job) {
|
||||
|
|
@ -113,7 +138,6 @@ export class DownloadService {
|
|||
}
|
||||
|
||||
async cancelJob(jobId: string): Promise<{ success: boolean; message: string }> {
|
||||
// Try the file download queue first (the original PR #554 path)
|
||||
const queue = this.queueService.getQueue(RunDownloadJob.queue)
|
||||
const job = await queue.getJob(jobId)
|
||||
|
||||
|
|
@ -121,7 +145,13 @@ export class DownloadService {
|
|||
return await this._cancelFileDownloadJob(jobId, job, queue)
|
||||
}
|
||||
|
||||
// Fall through to the model download queue
|
||||
const extractQueue = this.queueService.getQueue(RunExtractPmtilesJob.queue)
|
||||
const extractJob = await extractQueue.getJob(jobId)
|
||||
|
||||
if (extractJob) {
|
||||
return await this._cancelExtractJob(jobId, extractJob, extractQueue)
|
||||
}
|
||||
|
||||
const modelQueue = this.queueService.getQueue(DownloadModelJob.queue)
|
||||
const modelJob = await modelQueue.getJob(jobId)
|
||||
|
||||
|
|
@ -129,11 +159,37 @@ export class DownloadService {
|
|||
return await this._cancelModelDownloadJob(jobId, modelJob, modelQueue)
|
||||
}
|
||||
|
||||
// Not found in either queue
|
||||
return { success: true, message: 'Job not found (may have already completed)' }
|
||||
}
|
||||
|
||||
/** Cancel a content download (zim, map, pmtiles, etc.) — original PR #554 logic */
|
||||
private async _cancelExtractJob(
|
||||
jobId: string,
|
||||
job: Job<RunExtractPmtilesJobParams>,
|
||||
queue: Queue<RunExtractPmtilesJobParams>
|
||||
): Promise<{ success: boolean; message: string }> {
|
||||
const outputFilepath = job.data.outputFilepath
|
||||
|
||||
await RunExtractPmtilesJob.signalCancel(jobId)
|
||||
|
||||
// Same-process fallback when worker and API share a process
|
||||
RunExtractPmtilesJob.childProcesses.get(jobId)?.kill('SIGTERM')
|
||||
RunExtractPmtilesJob.childProcesses.delete(jobId)
|
||||
|
||||
await this._pollForTerminalState(job, jobId)
|
||||
await this._removeJobWithLockFallback(job, queue, RunExtractPmtilesJob.queue, jobId)
|
||||
|
||||
if (outputFilepath) {
|
||||
try {
|
||||
await deleteFileIfExists(outputFilepath)
|
||||
} catch {
|
||||
// File may not exist yet (subprocess may not have opened it)
|
||||
}
|
||||
}
|
||||
|
||||
return { success: true, message: 'Extract cancelled and partial file deleted' }
|
||||
}
|
||||
|
||||
/** Cancel a content download (zim, map, pmtiles, etc.) */
|
||||
private async _cancelFileDownloadJob(
|
||||
jobId: string,
|
||||
job: any,
|
||||
|
|
|
|||
|
|
@ -16,11 +16,35 @@ import {
|
|||
import { join, resolve, sep } from 'path'
|
||||
import urlJoin from 'url-join'
|
||||
import { RunDownloadJob } from '#jobs/run_download_job'
|
||||
import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { assertNotPrivateUrl } from '#validators/common'
|
||||
import InstalledResource from '#models/installed_resource'
|
||||
import { CollectionManifestService } from './collection_manifest_service.js'
|
||||
import type { CollectionWithStatus, MapsSpec } from '../../types/collections.js'
|
||||
import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps.js'
|
||||
import {
|
||||
EXTRACT_DEFAULT_MAX_ZOOM,
|
||||
EXTRACT_MAX_ZOOM,
|
||||
EXTRACT_MIN_ZOOM,
|
||||
PMTILES_BINARY_PATH,
|
||||
WORLD_BASEMAP_FILENAME,
|
||||
WORLD_BASEMAP_MAX_ZOOM,
|
||||
WORLD_BASEMAP_SOURCE_NAME,
|
||||
buildPmtilesExtractArgs,
|
||||
} from '../../constants/map_regions.js'
|
||||
import { CountriesService } from './countries_service.js'
|
||||
import { execFile } from 'child_process'
|
||||
import { createHash, randomBytes } from 'crypto'
|
||||
import { tmpdir } from 'os'
|
||||
import { promisify } from 'util'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const DRY_RUN_TIMEOUT_MS = 60_000
|
||||
const DRY_RUN_MAX_BUFFER = 256 * 1024
|
||||
// Real extract of z0-5 world tiles; generous to tolerate slow/metered links
|
||||
// since a failure leaves the map grey for uncovered regions.
|
||||
const WORLD_BASEMAP_EXTRACT_TIMEOUT_MS = 5 * 60_000
|
||||
|
||||
const PROTOMAPS_BUILDS_METADATA_URL = 'https://build-metadata.protomaps.dev/builds.json'
|
||||
const PROTOMAPS_BUILD_BASE_URL = 'https://build.protomaps.com'
|
||||
|
|
@ -53,10 +77,15 @@ export class MapService implements IMapService {
|
|||
private readonly baseAssetsTarFile = 'base-assets.tar.gz'
|
||||
private readonly baseDirPath = join(process.cwd(), this.mapStoragePath)
|
||||
private baseAssetsExistCache: boolean | null = null
|
||||
private worldBasemapReady = false
|
||||
private worldBasemapInFlight: Promise<void> | null = null
|
||||
|
||||
async listRegions() {
|
||||
const files = (await this.listAllMapStorageItems()).filter(
|
||||
(item) => item.type === 'file' && item.name.endsWith('.pmtiles')
|
||||
(item) =>
|
||||
item.type === 'file' &&
|
||||
item.name.endsWith('.pmtiles') &&
|
||||
item.name !== WORLD_BASEMAP_FILENAME
|
||||
)
|
||||
|
||||
return {
|
||||
|
|
@ -327,11 +356,76 @@ export class MapService implements IMapService {
|
|||
|
||||
async ensureBaseAssets(): Promise<boolean> {
|
||||
const exists = await this.checkBaseAssetsExist()
|
||||
if (exists) {
|
||||
return true
|
||||
if (!exists) {
|
||||
const downloaded = await this.downloadBaseAssets()
|
||||
if (!downloaded) return false
|
||||
}
|
||||
|
||||
return await this.downloadBaseAssets()
|
||||
try {
|
||||
await this.ensureWorldBasemap()
|
||||
} catch (err) {
|
||||
logger.warn(`[MapService] World basemap setup failed, continuing without it: ${err}`)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a low-zoom global basemap once so the map isn't grey outside a
|
||||
* regional extract's polygon. Cheap (~15 MB, a handful of HTTP range
|
||||
* requests) and layered underneath regional sources at render time.
|
||||
*
|
||||
* Memoizes success in-process, and de-duplicates concurrent callers via a
|
||||
* shared in-flight promise so two simultaneous `/maps` requests on a cold
|
||||
* start don't both launch `pmtiles extract` against the same output path.
|
||||
*/
|
||||
private async ensureWorldBasemap(): Promise<void> {
|
||||
if (this.worldBasemapReady) return
|
||||
if (this.worldBasemapInFlight) return this.worldBasemapInFlight
|
||||
this.worldBasemapInFlight = this._setupWorldBasemap().finally(() => {
|
||||
this.worldBasemapInFlight = null
|
||||
})
|
||||
return this.worldBasemapInFlight
|
||||
}
|
||||
|
||||
private async _setupWorldBasemap(): Promise<void> {
|
||||
const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
|
||||
const filepath = resolve(join(basePath, WORLD_BASEMAP_FILENAME))
|
||||
if (!filepath.startsWith(basePath + sep)) {
|
||||
throw new Error('Invalid world basemap path')
|
||||
}
|
||||
|
||||
await ensureDirectoryExists(basePath)
|
||||
|
||||
const existing = await getFileStatsIfExists(filepath)
|
||||
if (existing && Number(existing.size) > 0) {
|
||||
this.worldBasemapReady = true
|
||||
return
|
||||
}
|
||||
|
||||
const info = await this.getGlobalMapInfo()
|
||||
const args = buildPmtilesExtractArgs({
|
||||
sourceUrl: info.url,
|
||||
outputFilepath: filepath,
|
||||
maxzoom: WORLD_BASEMAP_MAX_ZOOM,
|
||||
downloadThreads: 4,
|
||||
})
|
||||
|
||||
logger.info(
|
||||
`[MapService] Extracting world basemap (z0-${WORLD_BASEMAP_MAX_ZOOM}) from ${info.url}`
|
||||
)
|
||||
try {
|
||||
await execFileAsync(PMTILES_BINARY_PATH, args, {
|
||||
timeout: WORLD_BASEMAP_EXTRACT_TIMEOUT_MS,
|
||||
maxBuffer: DRY_RUN_MAX_BUFFER,
|
||||
})
|
||||
this.worldBasemapReady = true
|
||||
} catch (err: any) {
|
||||
await deleteFileIfExists(filepath)
|
||||
throw new Error(
|
||||
`pmtiles extract for world basemap failed: ${err.message}. stderr: ${err.stderr ?? ''}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private async checkBaseAssetsExist(useCache: boolean = true): Promise<boolean> {
|
||||
|
|
@ -367,6 +461,19 @@ export class MapService implements IMapService {
|
|||
const sources: BaseStylesFile['sources'][] = []
|
||||
const baseUrl = this.getPublicFileBaseUrl(host, 'pmtiles', protocol)
|
||||
|
||||
// World basemap goes first so its layers render underneath regional extracts.
|
||||
// Only emitted when ensureWorldBasemap() succeeded — otherwise the style would
|
||||
// reference a file that doesn't exist and produce 404s on every tile request.
|
||||
if (this.worldBasemapReady) {
|
||||
const worldSource: BaseStylesFile['sources'] = {}
|
||||
worldSource[WORLD_BASEMAP_SOURCE_NAME] = {
|
||||
type: 'vector',
|
||||
attribution: PMTILES_ATTRIBUTION,
|
||||
url: `pmtiles://${urlJoin(baseUrl, WORLD_BASEMAP_FILENAME)}`,
|
||||
}
|
||||
sources.push(worldSource)
|
||||
}
|
||||
|
||||
for (const region of regions) {
|
||||
if (region.type === 'file' && region.name.endsWith('.pmtiles')) {
|
||||
// Strip .pmtiles and date suffix (e.g. "alaska_2025-12" -> "alaska") for stable source names
|
||||
|
|
@ -489,12 +596,206 @@ export class MapService implements IMapService {
|
|||
}
|
||||
}
|
||||
|
||||
async listCountries(): Promise<Country[]> {
|
||||
return CountriesService.getInstance().list()
|
||||
}
|
||||
|
||||
async listCountryGroups(): Promise<CountryGroup[]> {
|
||||
return CountriesService.getInstance().listGroups()
|
||||
}
|
||||
|
||||
async extractPreflight(params: {
|
||||
countries: CountryCode[]
|
||||
maxzoom?: number
|
||||
}): Promise<MapExtractPreflight> {
|
||||
this.validateMaxzoom(params.maxzoom)
|
||||
const countries = await CountriesService.getInstance().resolveCodes(params.countries)
|
||||
const regionFilepath = await CountriesService.getInstance().writeRegionFile(countries)
|
||||
const info = await this.getGlobalMapInfo()
|
||||
return this.runDryRun(info, regionFilepath, params.maxzoom)
|
||||
}
|
||||
|
||||
private async runDryRun(
|
||||
info: { url: string; date: string; key: string },
|
||||
regionFilepath: string,
|
||||
maxzoom?: number
|
||||
): Promise<MapExtractPreflight> {
|
||||
const dryRunOutput = join(tmpdir(), `pmtiles-dry-run-${randomBytes(6).toString('hex')}.pmtiles`)
|
||||
const args = buildPmtilesExtractArgs({
|
||||
sourceUrl: info.url,
|
||||
outputFilepath: dryRunOutput,
|
||||
regionFilepath,
|
||||
maxzoom,
|
||||
dryRun: true,
|
||||
})
|
||||
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
try {
|
||||
const result = await execFileAsync(PMTILES_BINARY_PATH, args, {
|
||||
timeout: DRY_RUN_TIMEOUT_MS,
|
||||
maxBuffer: DRY_RUN_MAX_BUFFER,
|
||||
})
|
||||
stdout = result.stdout
|
||||
stderr = result.stderr
|
||||
} catch (err: any) {
|
||||
throw new Error(
|
||||
`pmtiles extract --dry-run failed: ${err.message}. stderr: ${err.stderr ?? ''}`
|
||||
)
|
||||
}
|
||||
|
||||
const parsed = this.parseDryRunOutput(stdout + '\n' + stderr)
|
||||
|
||||
return {
|
||||
tiles: parsed.tiles,
|
||||
bytes: parsed.bytes,
|
||||
source: { url: info.url, date: info.date, key: info.key },
|
||||
}
|
||||
}
|
||||
|
||||
async extractRegion(params: {
|
||||
countries: CountryCode[]
|
||||
maxzoom?: number
|
||||
label?: string
|
||||
estimatedBytes?: number
|
||||
}): Promise<{ filename: string; jobId?: string }> {
|
||||
this.validateMaxzoom(params.maxzoom)
|
||||
const countriesService = CountriesService.getInstance()
|
||||
const countries = await countriesService.resolveCodes(params.countries)
|
||||
const regionFilepath = await countriesService.writeRegionFile(countries)
|
||||
const maxzoom = params.maxzoom ?? EXTRACT_DEFAULT_MAX_ZOOM
|
||||
|
||||
const [baseAssetsExist, info, groups] = await Promise.all([
|
||||
this.ensureBaseAssets(),
|
||||
this.getGlobalMapInfo(),
|
||||
countriesService.listGroups(),
|
||||
])
|
||||
if (!baseAssetsExist) {
|
||||
throw new Error(
|
||||
'Base map assets are missing and could not be downloaded. Please check your connection and try again.'
|
||||
)
|
||||
}
|
||||
|
||||
const groupMatch = findExactGroupMatch(countries, groups)
|
||||
const slug = this.buildRegionSlug(countries, groupMatch)
|
||||
const dateSlug = info.key.replace('.pmtiles', '')
|
||||
const filename = `${slug}_${dateSlug}_z${maxzoom}.pmtiles`
|
||||
const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
|
||||
const filepath = resolve(join(basePath, filename))
|
||||
|
||||
if (!filepath.startsWith(basePath + sep)) {
|
||||
throw new Error('Invalid filename')
|
||||
}
|
||||
|
||||
let estimatedBytes = params.estimatedBytes ?? 0
|
||||
if (estimatedBytes === 0) {
|
||||
try {
|
||||
const preflight = await this.runDryRun(info, regionFilepath, maxzoom)
|
||||
estimatedBytes = preflight.bytes
|
||||
} catch (err) {
|
||||
logger.warn(`[MapService] extractRegion preflight failed, proceeding without estimate: ${err}`)
|
||||
}
|
||||
}
|
||||
|
||||
const title = params.label ?? this.buildRegionTitle(countries, groupMatch)
|
||||
|
||||
const result = await RunExtractPmtilesJob.dispatch({
|
||||
sourceUrl: info.url,
|
||||
outputFilepath: filepath,
|
||||
regionFilepath,
|
||||
maxzoom,
|
||||
estimatedBytes,
|
||||
filetype: 'map',
|
||||
title,
|
||||
resourceMetadata: {
|
||||
resource_id: slug,
|
||||
version: dateSlug,
|
||||
collection_ref: null,
|
||||
},
|
||||
})
|
||||
|
||||
if (!result.job) {
|
||||
throw new Error('Failed to dispatch extract job')
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`[MapService] Dispatched extract job ${result.job.id} for ${filename} ` +
|
||||
`(countries=[${countries.join(',')}] maxzoom=${maxzoom} est=${estimatedBytes} bytes)`
|
||||
)
|
||||
|
||||
return {
|
||||
filename,
|
||||
jobId: result.job.id,
|
||||
}
|
||||
}
|
||||
|
||||
private buildRegionSlug(countries: CountryCode[], groupMatch: CountryGroup | null): string {
|
||||
if (groupMatch) return groupMatch.id
|
||||
if (countries.length === 1) return countries[0].toLowerCase()
|
||||
const hash = createHash('sha1').update(countries.join(',')).digest('hex').slice(0, 8)
|
||||
return `custom-${hash}`
|
||||
}
|
||||
|
||||
private buildRegionTitle(countries: CountryCode[], groupMatch: CountryGroup | null): string {
|
||||
if (groupMatch) return groupMatch.name
|
||||
if (countries.length === 1) return countries[0]
|
||||
if (countries.length <= 3) return countries.join(', ')
|
||||
return `${countries.slice(0, 2).join(', ')} +${countries.length - 2} more`
|
||||
}
|
||||
|
||||
private validateMaxzoom(maxzoom: number | undefined): void {
|
||||
if (typeof maxzoom !== 'number') return
|
||||
if (
|
||||
!Number.isInteger(maxzoom) ||
|
||||
maxzoom < EXTRACT_MIN_ZOOM ||
|
||||
maxzoom > EXTRACT_MAX_ZOOM
|
||||
) {
|
||||
throw new Error(
|
||||
`maxzoom must be an integer in [${EXTRACT_MIN_ZOOM}, ${EXTRACT_MAX_ZOOM}]`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// go-pmtiles output format isn't stable across versions — parse loosely and
|
||||
// fall back to zeros. The extract can still proceed without an estimate.
|
||||
private parseDryRunOutput(output: string): { tiles: number; bytes: number } {
|
||||
let bytes = 0
|
||||
let tiles = 0
|
||||
|
||||
const byteLine = output.match(/archive\s+size\s+of\s+([\d,.]+)\s*(B|KB|MB|GB|TB|bytes?)?/i)
|
||||
if (byteLine) {
|
||||
const raw = parseFloat(byteLine[1].replace(/,/g, ''))
|
||||
const unit = (byteLine[2] ?? 'B').toUpperCase()
|
||||
const multipliers: Record<string, number> = {
|
||||
B: 1,
|
||||
BYTE: 1,
|
||||
BYTES: 1,
|
||||
KB: 1_000,
|
||||
MB: 1_000_000,
|
||||
GB: 1_000_000_000,
|
||||
TB: 1_000_000_000_000,
|
||||
}
|
||||
bytes = Math.round(raw * (multipliers[unit] ?? 1))
|
||||
}
|
||||
|
||||
const tileLine = output.match(/(?:tiles\s+to\s+extract|tiles)[^\d]*([\d,]+)/i)
|
||||
if (tileLine) {
|
||||
tiles = parseInt(tileLine[1].replace(/,/g, ''), 10) || 0
|
||||
}
|
||||
|
||||
return { tiles, bytes }
|
||||
}
|
||||
|
||||
async delete(file: string): Promise<void> {
|
||||
let fileName = file
|
||||
if (!fileName.endsWith('.pmtiles')) {
|
||||
fileName += '.pmtiles'
|
||||
}
|
||||
|
||||
if (fileName === WORLD_BASEMAP_FILENAME) {
|
||||
throw new Error('The world basemap cannot be deleted')
|
||||
}
|
||||
|
||||
const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
|
||||
const fullPath = resolve(join(basePath, fileName))
|
||||
|
||||
|
|
@ -573,3 +874,16 @@ export class MapService implements IMapService {
|
|||
return baseUrl
|
||||
}
|
||||
}
|
||||
|
||||
function findExactGroupMatch(
|
||||
countries: CountryCode[],
|
||||
groups: CountryGroup[]
|
||||
): CountryGroup | null {
|
||||
return (
|
||||
groups.find(
|
||||
(g) =>
|
||||
g.countries.length === countries.length &&
|
||||
g.countries.every((c, i) => c === countries[i])
|
||||
) ?? null
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import {
|
|||
} from '../../types/system.js'
|
||||
import { SERVICE_NAMES } from '../../constants/service_names.js'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import path, { join } from 'node:path'
|
||||
import { getAllFilesystems, getFile } from '../utils/fs.js'
|
||||
import axios from 'axios'
|
||||
|
|
@ -72,6 +73,61 @@ export class SystemService {
|
|||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe Ollama startup logs for the canonical "inference compute" line that records
|
||||
* which compute backend was selected. This catches silent CPU fallback (e.g. when
|
||||
* /dev/kfd is mounted but ROCm initialization fails, or NVML dies after an update)
|
||||
* which the older nvidia-smi exec probe could not detect.
|
||||
*
|
||||
* Returns the parsed library, GPU model name, and VRAM in MiB, or null when:
|
||||
* - the Ollama container is not running
|
||||
* - the line has not been emitted (Ollama still starting up)
|
||||
* - logs show CPU-only operation (no GPU detected)
|
||||
*/
|
||||
async getOllamaInferenceComputeFromLogs(): Promise<{
|
||||
library: 'CUDA' | 'ROCm'
|
||||
name: string
|
||||
vramMiB: number
|
||||
} | null> {
|
||||
try {
|
||||
const containers = await this.dockerService.docker.listContainers({ all: false })
|
||||
const ollamaContainer = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`))
|
||||
if (!ollamaContainer) return null
|
||||
|
||||
const container = this.dockerService.docker.getContainer(ollamaContainer.Id)
|
||||
const buf = (await container.logs({
|
||||
stdout: true,
|
||||
stderr: true,
|
||||
tail: 500,
|
||||
follow: false,
|
||||
})) as unknown as Buffer
|
||||
const logs = buf.toString('utf8')
|
||||
|
||||
const lines = logs.split('\n').filter((l) => l.includes('msg="inference compute"'))
|
||||
if (lines.length === 0) return null
|
||||
|
||||
const lastLine = lines[lines.length - 1]
|
||||
const libraryMatch = lastLine.match(/library=(CUDA|ROCm)/)
|
||||
if (!libraryMatch) return null
|
||||
|
||||
const descMatch = lastLine.match(/description="([^"]+)"/)
|
||||
const totalMatch = lastLine.match(/total="([0-9.]+)\s*GiB"/)
|
||||
|
||||
return {
|
||||
library: libraryMatch[1] as 'CUDA' | 'ROCm',
|
||||
name:
|
||||
descMatch?.[1] ||
|
||||
(libraryMatch[1] === 'CUDA' ? 'NVIDIA GPU' : 'AMD GPU'),
|
||||
vramMiB: totalMatch ? Math.round(Number.parseFloat(totalMatch[1]) * 1024) : 0,
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`[SystemService] Failed to probe Ollama logs for inference compute line: ${error instanceof Error ? error.message : error}`
|
||||
)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
async getNvidiaSmiInfo(): Promise<
|
||||
| Array<{ vendor: string; model: string; vram: number }>
|
||||
| { error: string }
|
||||
|
|
@ -317,10 +373,14 @@ export class SystemService {
|
|||
logger.error('Error reading disk info file:', error)
|
||||
}
|
||||
|
||||
// GPU health tracking — detect when host has NVIDIA GPU but Ollama can't access it
|
||||
// GPU health tracking — detect when host has a GPU runtime but Ollama can't access it.
|
||||
// Primary probe: parse Ollama's "inference compute" startup log line for both NVIDIA
|
||||
// and AMD. Secondary probe (NVIDIA only): nvidia-smi exec, retained as a fallback for
|
||||
// hardware enrichment when log parsing has not yet captured a startup line.
|
||||
let gpuHealth: GpuHealthStatus = {
|
||||
status: 'no_gpu',
|
||||
hasNvidiaRuntime: false,
|
||||
hasRocmRuntime: false,
|
||||
ollamaGpuAccessible: false,
|
||||
}
|
||||
|
||||
|
|
@ -340,27 +400,51 @@ export class SystemService {
|
|||
}
|
||||
|
||||
// If si.graphics() returned no controllers (common inside Docker),
|
||||
// fall back to nvidia runtime + nvidia-smi detection
|
||||
// fall back to runtime + Ollama log probe to figure out what's accessible.
|
||||
if (!graphics.controllers || graphics.controllers.length === 0) {
|
||||
const runtimes = dockerInfo.Runtimes || {}
|
||||
if ('nvidia' in runtimes) {
|
||||
gpuHealth.hasNvidiaRuntime = true
|
||||
const nvidiaInfo = await this.getNvidiaSmiInfo()
|
||||
if (Array.isArray(nvidiaInfo)) {
|
||||
graphics.controllers = nvidiaInfo.map((gpu) => ({
|
||||
model: gpu.model,
|
||||
vendor: gpu.vendor,
|
||||
bus: '',
|
||||
vram: gpu.vram,
|
||||
vramDynamic: false, // assume false here, we don't actually use this field for our purposes.
|
||||
}))
|
||||
gpuHealth.hasNvidiaRuntime = 'nvidia' in runtimes
|
||||
|
||||
// AMD doesn't register a Docker runtime. Detection sources, in priority order:
|
||||
// 1. KV 'gpu.type' (set by DockerService._detectGPUType after first Ollama install)
|
||||
// 2. Marker file at /app/storage/.nomad-gpu-type (written by install_nomad.sh)
|
||||
// The marker file matters because the System page should reflect AMD presence
|
||||
// even before AI Assistant has been installed for the first time.
|
||||
let savedGpuType: string | null | undefined = await KVStore.getValue('gpu.type') as string | undefined
|
||||
if (!savedGpuType) {
|
||||
try {
|
||||
savedGpuType = (await readFile('/app/storage/.nomad-gpu-type', 'utf8')).trim()
|
||||
} catch {}
|
||||
}
|
||||
const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration')
|
||||
const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false'
|
||||
gpuHealth.hasRocmRuntime = savedGpuType === 'amd' && amdAccelerationEnabled
|
||||
|
||||
if (gpuHealth.hasNvidiaRuntime || gpuHealth.hasRocmRuntime) {
|
||||
gpuHealth.gpuVendor = gpuHealth.hasNvidiaRuntime ? 'nvidia' : 'amd'
|
||||
|
||||
// Primary probe: Ollama log parsing — works for both vendors and catches silent fallback
|
||||
const logInfo = await this.getOllamaInferenceComputeFromLogs()
|
||||
if (logInfo) {
|
||||
graphics.controllers = [
|
||||
{
|
||||
model: logInfo.name,
|
||||
vendor: logInfo.library === 'CUDA' ? 'NVIDIA' : 'AMD',
|
||||
bus: '',
|
||||
vram: logInfo.vramMiB,
|
||||
vramDynamic: false,
|
||||
},
|
||||
]
|
||||
gpuHealth.status = 'ok'
|
||||
gpuHealth.ollamaGpuAccessible = true
|
||||
} else if (nvidiaInfo === 'OLLAMA_NOT_FOUND') {
|
||||
// No local Ollama container — check if a remote Ollama URL is configured
|
||||
const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
|
||||
if (externalOllamaGpu) {
|
||||
graphics.controllers = externalOllamaGpu.map((gpu) => ({
|
||||
} else if (gpuHealth.hasNvidiaRuntime) {
|
||||
// NVIDIA secondary path: nvidia-smi exec preserves prior behavior when
|
||||
// the log parser hasn't seen a startup line yet (e.g. log rotation,
|
||||
// very fresh container). Distinguishes "no Ollama container" from
|
||||
// "container exists but GPU broken".
|
||||
const nvidiaInfo = await this.getNvidiaSmiInfo()
|
||||
if (Array.isArray(nvidiaInfo)) {
|
||||
graphics.controllers = nvidiaInfo.map((gpu) => ({
|
||||
model: gpu.model,
|
||||
vendor: gpu.vendor,
|
||||
bus: '',
|
||||
|
|
@ -369,25 +453,66 @@ export class SystemService {
|
|||
}))
|
||||
gpuHealth.status = 'ok'
|
||||
gpuHealth.ollamaGpuAccessible = true
|
||||
} else if (nvidiaInfo === 'OLLAMA_NOT_FOUND') {
|
||||
const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
|
||||
if (externalOllamaGpu) {
|
||||
graphics.controllers = externalOllamaGpu.map((gpu) => ({
|
||||
model: gpu.model,
|
||||
vendor: gpu.vendor,
|
||||
bus: '',
|
||||
vram: gpu.vram,
|
||||
vramDynamic: false,
|
||||
}))
|
||||
gpuHealth.status = 'ok'
|
||||
gpuHealth.ollamaGpuAccessible = true
|
||||
} else {
|
||||
gpuHealth.status = 'ollama_not_installed'
|
||||
}
|
||||
} else {
|
||||
gpuHealth.status = 'ollama_not_installed'
|
||||
const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
|
||||
if (externalOllamaGpu) {
|
||||
graphics.controllers = externalOllamaGpu.map((gpu) => ({
|
||||
model: gpu.model,
|
||||
vendor: gpu.vendor,
|
||||
bus: '',
|
||||
vram: gpu.vram,
|
||||
vramDynamic: false,
|
||||
}))
|
||||
gpuHealth.status = 'ok'
|
||||
gpuHealth.ollamaGpuAccessible = true
|
||||
} else {
|
||||
gpuHealth.status = 'passthrough_failed'
|
||||
logger.warn(
|
||||
`NVIDIA runtime detected but GPU passthrough failed: ${typeof nvidiaInfo === 'string' ? nvidiaInfo : JSON.stringify(nvidiaInfo)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
|
||||
if (externalOllamaGpu) {
|
||||
graphics.controllers = externalOllamaGpu.map((gpu) => ({
|
||||
model: gpu.model,
|
||||
vendor: gpu.vendor,
|
||||
bus: '',
|
||||
vram: gpu.vram,
|
||||
vramDynamic: false,
|
||||
}))
|
||||
gpuHealth.status = 'ok'
|
||||
gpuHealth.ollamaGpuAccessible = true
|
||||
// AMD path: no nvidia-smi equivalent worth running — log parser is authoritative.
|
||||
// Distinguish "Ollama not running" from "Ollama running but no GPU log line".
|
||||
const containers = await this.dockerService.docker.listContainers({ all: false })
|
||||
const ollamaRunning = containers.some((c) =>
|
||||
c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`)
|
||||
)
|
||||
if (!ollamaRunning) {
|
||||
const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
|
||||
if (externalOllamaGpu) {
|
||||
graphics.controllers = externalOllamaGpu.map((gpu) => ({
|
||||
model: gpu.model,
|
||||
vendor: gpu.vendor,
|
||||
bus: '',
|
||||
vram: gpu.vram,
|
||||
vramDynamic: false,
|
||||
}))
|
||||
gpuHealth.status = 'ok'
|
||||
gpuHealth.ollamaGpuAccessible = true
|
||||
} else {
|
||||
gpuHealth.status = 'ollama_not_installed'
|
||||
}
|
||||
} else {
|
||||
gpuHealth.status = 'passthrough_failed'
|
||||
logger.warn(
|
||||
`NVIDIA runtime detected but GPU passthrough failed: ${typeof nvidiaInfo === 'string' ? nvidiaInfo : JSON.stringify(nvidiaInfo)}`
|
||||
'AMD GPU detected but Ollama logs show no ROCm initialization — passthrough or HSA override may have failed'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import {
|
|||
RemoteZimFileEntry,
|
||||
} from '../../types/zim.js'
|
||||
import axios from 'axios'
|
||||
import * as cheerio from 'cheerio'
|
||||
import { XMLParser } from 'fast-xml-parser'
|
||||
import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from '../../util/zim.js'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
|
@ -27,6 +28,8 @@ import { SERVICE_NAMES } from '../../constants/service_names.js'
|
|||
import { CollectionManifestService } from './collection_manifest_service.js'
|
||||
import { KiwixLibraryService } from './kiwix_library_service.js'
|
||||
import type { CategoryWithStatus } from '../../types/collections.js'
|
||||
import CustomLibrarySource from '#models/custom_library_source'
|
||||
import { assertNotPrivateUrl } from '#validators/common'
|
||||
|
||||
const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
|
||||
const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json'
|
||||
|
|
@ -587,25 +590,47 @@ export class ZimService {
|
|||
}
|
||||
|
||||
async onWikipediaDownloadComplete(url: string, success: boolean): Promise<void> {
|
||||
const filename = url.split('/').pop() || ''
|
||||
const selection = await this.getWikipediaSelection()
|
||||
|
||||
if (!selection || selection.url !== url) {
|
||||
logger.warn(`[ZimService] Wikipedia download complete callback for unknown URL: ${url}`)
|
||||
return
|
||||
// Determine which Wikipedia option this file belongs to by matching filename
|
||||
let matchedOptionId: string | null = null
|
||||
try {
|
||||
const options = await this.getWikipediaOptions()
|
||||
for (const opt of options) {
|
||||
if (opt.url && opt.url.split('/').pop() === filename) {
|
||||
matchedOptionId = opt.id
|
||||
break
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// If we can't fetch options, try to continue with existing selection
|
||||
}
|
||||
|
||||
if (success) {
|
||||
// Update status to installed
|
||||
selection.status = 'installed'
|
||||
await selection.save()
|
||||
// Update or create the selection record
|
||||
// Match by filename (not URL) so mirror downloads are recognized
|
||||
if (selection) {
|
||||
selection.option_id = matchedOptionId || selection.option_id
|
||||
selection.url = url
|
||||
selection.filename = filename
|
||||
selection.status = 'installed'
|
||||
await selection.save()
|
||||
} else {
|
||||
await WikipediaSelection.create({
|
||||
option_id: matchedOptionId || 'unknown',
|
||||
url: url,
|
||||
filename: filename,
|
||||
status: 'installed',
|
||||
})
|
||||
}
|
||||
|
||||
logger.info(`[ZimService] Wikipedia download completed successfully: ${selection.filename}`)
|
||||
logger.info(`[ZimService] Wikipedia download completed successfully: ${filename}`)
|
||||
|
||||
// Delete the old Wikipedia file if it exists and is different
|
||||
// We need to find what was previously installed
|
||||
// Delete old Wikipedia files (keep only the newly installed one)
|
||||
const existingFiles = await this.list()
|
||||
const wikipediaFiles = existingFiles.files.filter((f) =>
|
||||
f.name.startsWith('wikipedia_en_') && f.name !== selection.filename
|
||||
f.name.startsWith('wikipedia_en_') && f.name !== filename
|
||||
)
|
||||
|
||||
for (const oldFile of wikipediaFiles) {
|
||||
|
|
@ -617,10 +642,137 @@ export class ZimService {
|
|||
}
|
||||
}
|
||||
} else {
|
||||
// Download failed - keep the selection record but mark as failed
|
||||
selection.status = 'failed'
|
||||
await selection.save()
|
||||
logger.error(`[ZimService] Wikipedia download failed for: ${selection.filename}`)
|
||||
// Download failed - update selection if it matches this file
|
||||
if (selection && (!selection.filename || selection.filename === filename)) {
|
||||
selection.status = 'failed'
|
||||
await selection.save()
|
||||
logger.error(`[ZimService] Wikipedia download failed for: ${filename}`)
|
||||
} else {
|
||||
logger.error(`[ZimService] Wikipedia download failed for: ${filename} (no matching selection)`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Custom library source management
|
||||
|
||||
async listCustomLibraries(): Promise<CustomLibrarySource[]> {
|
||||
return CustomLibrarySource.all()
|
||||
}
|
||||
|
||||
async addCustomLibrary(name: string, baseUrl: string): Promise<CustomLibrarySource> {
|
||||
const count = await CustomLibrarySource.query().count('* as total')
|
||||
const total = Number(count[0].$extras.total)
|
||||
if (total >= 10) {
|
||||
throw new Error('Maximum of 10 custom libraries allowed')
|
||||
}
|
||||
|
||||
// Ensure URL ends with /
|
||||
const normalizedUrl = baseUrl.endsWith('/') ? baseUrl : baseUrl + '/'
|
||||
|
||||
return CustomLibrarySource.create({
|
||||
name,
|
||||
base_url: normalizedUrl,
|
||||
})
|
||||
}
|
||||
|
||||
async removeCustomLibrary(id: number): Promise<void> {
|
||||
const source = await CustomLibrarySource.find(id)
|
||||
if (!source) {
|
||||
throw new Error('Custom library not found')
|
||||
}
|
||||
if (source.is_default) {
|
||||
throw new Error('Cannot remove a built-in mirror')
|
||||
}
|
||||
await source.delete()
|
||||
}
|
||||
|
||||
async browseLibraryUrl(url: string): Promise<{
|
||||
directories: { name: string; url: string }[]
|
||||
files: { name: string; url: string; size_bytes: number | null }[]
|
||||
}> {
|
||||
assertNotPrivateUrl(url)
|
||||
|
||||
const normalizedUrl = url.endsWith('/') ? url : url + '/'
|
||||
|
||||
const res = await axios.get(normalizedUrl, {
|
||||
responseType: 'text',
|
||||
timeout: 15000,
|
||||
headers: {
|
||||
'Accept': 'text/html',
|
||||
},
|
||||
})
|
||||
|
||||
const html: string = res.data
|
||||
const directories: { name: string; url: string }[] = []
|
||||
const files: { name: string; url: string; size_bytes: number | null }[] = []
|
||||
|
||||
const $ = cheerio.load(html)
|
||||
|
||||
$('a').each((_, el) => {
|
||||
const href = el.attribs?.href
|
||||
if (!href || href === '../' || href === './' || href === '/' || href.startsWith('?') || href.startsWith('#')) {
|
||||
return
|
||||
}
|
||||
if (href.startsWith('/') || href.startsWith('http://') || href.startsWith('https://')) {
|
||||
return
|
||||
}
|
||||
|
||||
if (href.endsWith('/')) {
|
||||
const dirName = decodeURIComponent(href.replace(/\/$/, ''))
|
||||
directories.push({
|
||||
name: dirName,
|
||||
url: new URL(href, normalizedUrl).toString(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if (href.endsWith('.zim')) {
|
||||
const fileName = decodeURIComponent(href)
|
||||
|
||||
// Apache/Nginx autoindex put the date + size in the text node directly
|
||||
// following </a> within a <pre>. Walk forward across text siblings until
|
||||
// we find a parseable size token.
|
||||
let trailingText = ''
|
||||
let sibling = el.next
|
||||
while (sibling && sibling.type === 'text') {
|
||||
trailingText += sibling.data
|
||||
if (/\n/.test(sibling.data)) break
|
||||
sibling = sibling.next
|
||||
}
|
||||
|
||||
files.push({
|
||||
name: fileName,
|
||||
url: new URL(href, normalizedUrl).toString(),
|
||||
size_bytes: this._parseListingSize(trailingText),
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
directories.sort((a, b) => a.name.localeCompare(b.name))
|
||||
files.sort((a, b) => a.name.localeCompare(b.name))
|
||||
|
||||
return { directories, files }
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a directory-listing size token out of the text that follows an anchor.
|
||||
* Apache renders e.g. ` 2024-01-15 10:30 5.1G`; Nginx renders raw bytes.
|
||||
* Returns bytes or null if no size token is found.
|
||||
*/
|
||||
private _parseListingSize(text: string): number | null {
|
||||
// Skip the date/time columns; grab the last numeric token (with optional suffix)
|
||||
// before a newline. Matches `5.1G`, `5368709120`, `1.2T`, etc.
|
||||
const sizeMatch = /([\d.]+\s*[KMGT]?B?|\d+)\s*$/i.exec(text.split('\n')[0].trim())
|
||||
if (!sizeMatch) return null
|
||||
|
||||
const sizeStr = sizeMatch[1].replace(/\s|B$/gi, '')
|
||||
const num = parseFloat(sizeStr)
|
||||
if (isNaN(num)) return null
|
||||
|
||||
if (/^\d+$/.test(sizeStr)) return num
|
||||
|
||||
const suffix = sizeStr.slice(-1).toUpperCase()
|
||||
const multipliers: Record<string, number> = { K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 }
|
||||
return multipliers[suffix] ? Math.round(num * multipliers[suffix]) : null
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -100,6 +100,7 @@ const resourceUpdateInfoBase = vine.object({
|
|||
installed_version: vine.string().trim(),
|
||||
latest_version: vine.string().trim().minLength(1),
|
||||
download_url: vine.string().url({ require_tld: false }).trim(),
|
||||
size_bytes: vine.number().positive().optional(),
|
||||
})
|
||||
|
||||
export const applyContentUpdateValidator = vine.compile(resourceUpdateInfoBase)
|
||||
|
|
@ -111,3 +112,31 @@ export const applyAllContentUpdatesValidator = vine.compile(
|
|||
.minLength(1),
|
||||
})
|
||||
)
|
||||
|
||||
// --- Map extract (regional pmtiles download) ---
|
||||
|
||||
// ISO 3166-1 alpha-2, 2 letters. Loose regex; CountriesService.resolveCodes
|
||||
// does the authoritative check against the polygon dataset.
|
||||
const countryCodeSchema = vine
|
||||
.string()
|
||||
.trim()
|
||||
.toUpperCase()
|
||||
.regex(/^[A-Z]{2}$/)
|
||||
|
||||
const countriesArraySchema = vine.array(countryCodeSchema).minLength(1).maxLength(300)
|
||||
|
||||
export const mapExtractPreflightValidator = vine.compile(
|
||||
vine.object({
|
||||
countries: countriesArraySchema.clone(),
|
||||
maxzoom: vine.number().min(0).max(15).optional(),
|
||||
})
|
||||
)
|
||||
|
||||
export const mapExtractValidator = vine.compile(
|
||||
vine.object({
|
||||
countries: countriesArraySchema.clone(),
|
||||
maxzoom: vine.number().min(0).max(15).optional(),
|
||||
label: vine.string().trim().minLength(1).maxLength(64).optional(),
|
||||
estimatedBytes: vine.number().min(0).optional(),
|
||||
})
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,3 +7,30 @@ export const listRemoteZimValidator = vine.compile(
|
|||
query: vine.string().optional(),
|
||||
})
|
||||
)
|
||||
|
||||
export const addCustomLibraryValidator = vine.compile(
|
||||
vine.object({
|
||||
name: vine.string().trim().minLength(1).maxLength(100),
|
||||
base_url: vine
|
||||
.string()
|
||||
.url({ require_tld: false })
|
||||
.trim(),
|
||||
})
|
||||
)
|
||||
|
||||
export const browseLibraryValidator = vine.compile(
|
||||
vine.object({
|
||||
url: vine
|
||||
.string()
|
||||
.url({ require_tld: false })
|
||||
.trim(),
|
||||
})
|
||||
)
|
||||
|
||||
export const idParamValidator = vine.compile(
|
||||
vine.object({
|
||||
params: vine.object({
|
||||
id: vine.number(),
|
||||
}),
|
||||
})
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import type { CommandOptions } from '@adonisjs/core/types/ace'
|
|||
import { Worker } from 'bullmq'
|
||||
import queueConfig from '#config/queue'
|
||||
import { RunDownloadJob } from '#jobs/run_download_job'
|
||||
import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job'
|
||||
import { DownloadModelJob } from '#jobs/download_model_job'
|
||||
import { RunBenchmarkJob } from '#jobs/run_benchmark_job'
|
||||
import { EmbedFileJob } from '#jobs/embed_file_job'
|
||||
|
|
@ -126,6 +127,7 @@ export default class QueueWork extends BaseCommand {
|
|||
const queues = new Map<string, string>()
|
||||
|
||||
handlers.set(RunDownloadJob.key, new RunDownloadJob())
|
||||
handlers.set(RunExtractPmtilesJob.key, new RunExtractPmtilesJob())
|
||||
handlers.set(DownloadModelJob.key, new DownloadModelJob())
|
||||
handlers.set(RunBenchmarkJob.key, new RunBenchmarkJob())
|
||||
handlers.set(EmbedFileJob.key, new EmbedFileJob())
|
||||
|
|
@ -133,6 +135,7 @@ export default class QueueWork extends BaseCommand {
|
|||
handlers.set(CheckServiceUpdatesJob.key, new CheckServiceUpdatesJob())
|
||||
|
||||
queues.set(RunDownloadJob.key, RunDownloadJob.queue)
|
||||
queues.set(RunExtractPmtilesJob.key, RunExtractPmtilesJob.queue)
|
||||
queues.set(DownloadModelJob.key, DownloadModelJob.queue)
|
||||
queues.set(RunBenchmarkJob.key, RunBenchmarkJob.queue)
|
||||
queues.set(EmbedFileJob.key, EmbedFileJob.queue)
|
||||
|
|
@ -149,6 +152,9 @@ export default class QueueWork extends BaseCommand {
|
|||
private getConcurrencyForQueue(queueName: string): number {
|
||||
const concurrencyMap: Record<string, number> = {
|
||||
[RunDownloadJob.queue]: 3,
|
||||
// pmtiles extract hits the Protomaps CDN with many parallel range reads per job;
|
||||
// cap concurrency at 2 so a second extract doesn't starve the first.
|
||||
[RunExtractPmtilesJob.queue]: 2,
|
||||
[DownloadModelJob.queue]: 2, // Lower concurrency for resource-intensive model downloads
|
||||
[RunBenchmarkJob.queue]: 1, // Run benchmarks one at a time for accurate results
|
||||
[EmbedFileJob.queue]: 2, // Lower concurrency for embedding jobs, can be resource intensive
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
export const PMTILES_BINARY_PATH = '/usr/local/bin/pmtiles'
|
||||
|
||||
// Clamp these so a user can't ask for nonsense that never extracts
|
||||
export const EXTRACT_MIN_ZOOM = 0
|
||||
export const EXTRACT_MAX_ZOOM = 15
|
||||
export const EXTRACT_DEFAULT_MAX_ZOOM = 15
|
||||
|
||||
// Low-zoom global fallback extracted once during base-asset setup (~15 MB). Layered
|
||||
// underneath regional extracts so the map isn't grey outside a region's polygon.
|
||||
export const WORLD_BASEMAP_FILENAME = 'world.pmtiles'
|
||||
export const WORLD_BASEMAP_MAX_ZOOM = 5
|
||||
export const WORLD_BASEMAP_SOURCE_NAME = 'world'
|
||||
|
||||
export interface PmtilesExtractArgOptions {
|
||||
sourceUrl: string
|
||||
outputFilepath: string
|
||||
regionFilepath?: string
|
||||
maxzoom?: number
|
||||
dryRun?: boolean
|
||||
downloadThreads?: number
|
||||
overfetch?: number
|
||||
}
|
||||
|
||||
export function buildPmtilesExtractArgs(opts: PmtilesExtractArgOptions): string[] {
|
||||
const args = ['extract', opts.sourceUrl, opts.outputFilepath]
|
||||
if (opts.regionFilepath) args.push(`--region=${opts.regionFilepath}`)
|
||||
if (typeof opts.maxzoom === 'number') args.push(`--maxzoom=${opts.maxzoom}`)
|
||||
if (opts.dryRun) args.push('--dry-run')
|
||||
if (typeof opts.downloadThreads === 'number') args.push(`--download-threads=${opts.downloadThreads}`)
|
||||
if (typeof opts.overfetch === 'number') args.push(`--overfetch=${opts.overfetch}`)
|
||||
return args
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'custom_library_sources'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.increments('id').primary()
|
||||
table.string('name', 100).notNullable()
|
||||
table.string('base_url', 2048).notNullable()
|
||||
table.boolean('is_default').notNullable().defaultTo(false)
|
||||
table.timestamp('created_at').notNullable()
|
||||
table.timestamp('updated_at').notNullable()
|
||||
})
|
||||
|
||||
// Seed default Kiwix mirrors
|
||||
const now = new Date().toISOString().slice(0, 19).replace('T', ' ')
|
||||
const defaults = [
|
||||
{ name: 'Debian CDN (Global)', base_url: 'https://cdimage.debian.org/mirror/kiwix.org/zim/' },
|
||||
{ name: 'Your.org (US)', base_url: 'https://ftpmirror.your.org/pub/kiwix/zim/' },
|
||||
{ name: 'FAU Erlangen (DE)', base_url: 'https://ftp.fau.de/kiwix/zim/' },
|
||||
{ name: 'Dotsrc (DK)', base_url: 'https://mirrors.dotsrc.org/kiwix/zim/' },
|
||||
{ name: 'MirrorService (UK)', base_url: 'https://www.mirrorservice.org/sites/download.kiwix.org/zim/' },
|
||||
]
|
||||
|
||||
for (const d of defaults) {
|
||||
await this.defer(async (db) => {
|
||||
await db.table(this.tableName).insert({
|
||||
name: d.name,
|
||||
base_url: d.base_url,
|
||||
is_default: true,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
|
|
@ -153,8 +153,8 @@ ZIM files provide offline Wikipedia, books, and other content via Kiwix.
|
|||
| Method | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| GET | `/api/maps/markers` | List map markers |
|
||||
| POST | `/api/maps/markers` | Add map marker (body: {"name": "Test Marker", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) |
|
||||
| PATCH | `/api/maps/markers/{id}` | Update a map marker (body: {"name": "Test Marker", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) fields that don't change can be omitted|
|
||||
| POST | `/api/maps/markers` | Add map marker (body: {"name": "Test Marker", "notes": "Example note", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) |
|
||||
| PATCH | `/api/maps/markers/{id}` | Update a map marker (body: {"name": "Test Marker", "notes": "Example note", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) fields that don't change can be omitted|
|
||||
| DELETE | `/api/maps/markers/{id}` | Delete a map marker |
|
||||
|
||||
---
|
||||
|
|
|
|||
|
|
@ -2,21 +2,57 @@
|
|||
|
||||
## Unreleased
|
||||
|
||||
### Features
|
||||
- **AI Assistant**: Added improved support for AMD GPU acceleration for Ollama via ROCm + HSA override. Thanks @chriscrosstalk for the contribution!
|
||||
- **Content Explorer**: Added support for custom ZIM library sources and pre-seeded ZIM library mirrors in addition to the default Kiwix library. Thanks @chriscrosstalk for the contribution!
|
||||
- **Content Manager**: Content update sizes and downloads are now properly displayed in Active Downloads with progress bars and friendly names. Thanks @chriscrosstalk for the contribution!
|
||||
- **Maps**: Map regions can now be extracted and downloaded locally from PMTiles to avoid the need for a full global map download for users who only want specific regions. Thanks @bgauger for the contribution!
|
||||
|
||||
### Bug Fixes
|
||||
- **API**: Compression is now skipped for Server-Sent Events (SSE) responses to prevent issues with streaming endpoints. Thanks @chriscrosstalk for the fix!
|
||||
- **Maps**: Fixed logic issues with the global map banner display. Thanks @Gujiassh for the fix!
|
||||
- **Maps**: The selected map file is now properly deleted after confirming the action in the UI. Thanks @cuyua9 for the fix!
|
||||
- **System**: Fixed an issue where the a pending update could still be indicated in the UI even after the system was updated successfully. Thanks @jakeaturner for the fix!
|
||||
|
||||
### Improvements
|
||||
- **Build**: The Command Center image now uses the VERSION build arg to write `app/version.json` with the current version for improved version tracking and debugging, even in RC environments. Thanks @chriscrosstalk for the contribution!
|
||||
- **Content Manager**: Added a sortable file size column to the ZIM files table in the Content Manager for easier management of storage space. Thanks @chriscrosstalk for the contribution!
|
||||
- **Dependencies**: All package.json dependencies have been pinned to specific versions to ensure stability and reduce the risk of unexpected breaking changes/supply-chain compromises from upstream packages. Thanks @jakeaturner for the contribution!
|
||||
- **Dependencies**: Updated various dependencies to close security vulnerabilities and improve stability
|
||||
- **Docs**: Update CONTIRBUTING.md to require an issue to be opened before submitting a PR for non-trivial changes to ensure proper discussion and review of proposed changes. Thanks @chriscrosstalk for the contribution!
|
||||
- **Docs**: Added the map markers endpoints to the API reference documentation. Thanks @kennethbrewer3 for the contribution!
|
||||
- **Docs**: Added a link to the new WSL2 install guide in the README and FAQ. Thanks @chriscrosstalk for the contribution!
|
||||
- **Install**: The install script now warns loudly if the user is attempting to install on a non-x86_64/amd64 platform to prevent unsupported installations and potential issues. Thanks @chriscrosstalk for the contribution!
|
||||
- **Maps**: The maps API endpoints now properly accept and validate notes, marker_type, and position data for map markers and persist them in the database for retrieval in the UI. Thanks @jrsphoto for the contribution!
|
||||
- **Maps**: The current coordinates of the mouse pointer can now be displayed in the map viewer for easier navigation and exploration. Thanks @kennethbrewer3 for the contribution!
|
||||
- **RAG**: NOMAD now properly passed `num_ctx` and truncation to the Ollama embedding endpoint to ensure that the context window of the model is best utilized for embeddings. Thanks @chriscrosstalk for the contribution!
|
||||
- **RAG**: Added a manual start button for Qdrant and a self-healing mechanism for Qdrant's restart-policy to ensure that the vector database is running properly for embedding and retrieval tasks. Thanks @hestela for the contribution!
|
||||
|
||||
## Version 1.31.1 - April 21, 2026
|
||||
|
||||
### Features
|
||||
|
||||
### Bug Fixes
|
||||
- **AI Assistant**: In-progress model downloads can now be cancelled properly and the progress UI now matches that of file downloads. Thanks @chriscrosstalk for the contribution!
|
||||
- **AI Assistant**: Fixed an issue where the AI Assistant settings page could crash if a model object did not have a details property. Thanks @hestela for the fix!
|
||||
- **AI Assistant**: Fixed an issue with non-embeddable files being queued for embedding and flooding logs with errors. Thanks @sbruschke for the bug report and @chriscrosstalk for the fix!
|
||||
- **AI Assistant**: Fixed an issue with ZIM batch embedding using the wrong batch count and causing remaining batches to be skipped. Thanks @sbruschke for the bug report and @chriscrosstalk for the fix!
|
||||
- **AI Assistant**: Fixed an issue with ZIM content extraction only extracting the first-level children of the article body and thus missing a lot of content. Thanks @sbruschke for the bug report and @chriscrosstalk for the fix!
|
||||
- **Disk Collector**: Improved reporting for NFS mount stats and display in the UI. Thanks @bgauger and @bravosierra99 for the contribution!
|
||||
- **Downloads**: Downloads are now staged to .tmp files and atomically renamed upon completion to prevent issues with incomplete/corrupt files. Thanks @artbird309 for the contribution!
|
||||
- **Downloads**: Removed a duplicate error listener and improved stability when handling Range requests for file downloads. Thanks @jakeaturner for the contribution!
|
||||
- **Downloads**: Added improved handling for corrupt ZIM file downloads and removed duplicate Ollama download logs. Thanks @aegisman for the contribution!
|
||||
- **Security**: Closed a potential SSRF vulnerability in the map file download functionality by implementing stricter URL validation and blocking private IP ranges. Thanks @LuisMIguelFurlanettoSousa for the fix!
|
||||
- **Security**: Sanitized error messages from the backend to prevent potential information disclosure. Thanks @LuisMIguelFurlanettoSousa for the fix!
|
||||
- **UI**: Fixed an issue with broken pagination for the Content Explorer that could cause some users to see a "No records found" message indefinitely. Thanks @johno10661 for the bug report and @chriscrosstalk for the fix!
|
||||
- **UI**: Fixed an issue where all storage devices could report as "NAS Storage" regardless of actual type. Thanks @bgauger for the fix!
|
||||
|
||||
### Improvements
|
||||
- **AI Assistant**: Now uses the currently loaded model for query rewriting and chat title generation for improved performance and consistency. Thanks @hestela for the contribution!
|
||||
- **AI Assistant**: When a remote Ollama URL is configured, the Command Center will now attempt to stop NOMAD's local Ollama container to free up resources and avoid confusion. Thanks @chriscrosstalk for the contribution!
|
||||
- **Dependencies**: Updated various dependencies to close security vulnerabilities and improve stability
|
||||
- **Docs**: Added a "Community Add-Ons" page to the documentation to highlight some of the amazing community contributions that have been made since launch. Thanks @chriscrosstalk for the contribution!
|
||||
- **Privacy**: Added the appropriate environment variable to disable telemetry for the Qdrant container. Note that this will only take effect on new installations of if the Qdrant container is force re-installed on existing installations. Thanks @berkdamerc for the find and @chriscrosstalk for the contribution!
|
||||
|
||||
## Version 1.31.0 - April 3, 2026
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,414 @@
|
|||
import { useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
import { IconCheck, IconSearch, IconX } from '@tabler/icons-react'
|
||||
import StyledModal, { StyledModalProps } from './StyledModal'
|
||||
import LoadingSpinner from './LoadingSpinner'
|
||||
import api from '~/lib/api'
|
||||
import { formatBytes } from '~/lib/util'
|
||||
import classNames from '~/lib/classNames'
|
||||
import {
|
||||
EXTRACT_DEFAULT_MAX_ZOOM,
|
||||
EXTRACT_MAX_ZOOM,
|
||||
EXTRACT_MIN_ZOOM,
|
||||
} from '../../constants/map_regions'
|
||||
import type {
|
||||
Country,
|
||||
CountryCode,
|
||||
CountryGroup,
|
||||
MapExtractPreflight,
|
||||
} from '../../types/maps'
|
||||
|
||||
export type CountryPickerModalProps = Omit<
|
||||
StyledModalProps,
|
||||
| 'onConfirm'
|
||||
| 'open'
|
||||
| 'confirmText'
|
||||
| 'cancelText'
|
||||
| 'confirmVariant'
|
||||
| 'children'
|
||||
| 'title'
|
||||
| 'large'
|
||||
> & {
|
||||
onDownloadStart?: () => void
|
||||
/** Filenames of pmtiles already on disk; used to badge already-installed countries. */
|
||||
installedFilenames?: string[]
|
||||
}
|
||||
|
||||
// Single-country extracts use the slug `{iso2 lowercase}_{dateSlug}_z{maxzoom}.pmtiles`,
|
||||
// matching MapService.buildRegionSlug (which lowercases the alpha-2 country code).
|
||||
// dateSlug comes from the upstream pmtiles key with `.pmtiles` stripped — currently
|
||||
// YYYYMMDD but we accept any digits/dashes. Group / custom filenames don't reverse-map
|
||||
// to country codes, so we skip them here.
|
||||
const SINGLE_COUNTRY_FILENAME_RE = /^([a-z]{2})_[\w-]+_z\d+\.pmtiles$/
|
||||
|
||||
const CountryPickerModal: React.FC<CountryPickerModalProps> = ({
|
||||
onDownloadStart,
|
||||
installedFilenames = [],
|
||||
...modalProps
|
||||
}) => {
|
||||
const [selected, setSelected] = useState<Set<CountryCode>>(new Set())
|
||||
const [search, setSearch] = useState('')
|
||||
const [maxzoom, setMaxzoom] = useState<number>(EXTRACT_DEFAULT_MAX_ZOOM)
|
||||
const [preflight, setPreflight] = useState<MapExtractPreflight | null>(null)
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [downloading, setDownloading] = useState(false)
|
||||
const [errorMessage, setErrorMessage] = useState<string | null>(null)
|
||||
const preflightRequestIdRef = useRef(0)
|
||||
|
||||
const { data: countries = [], isLoading: countriesLoading } = useQuery({
|
||||
queryKey: ['maps-countries'],
|
||||
queryFn: () => api.listCountries(),
|
||||
staleTime: Infinity,
|
||||
})
|
||||
|
||||
const { data: groups = [] } = useQuery({
|
||||
queryKey: ['maps-country-groups'],
|
||||
queryFn: () => api.listCountryGroups(),
|
||||
staleTime: Infinity,
|
||||
})
|
||||
|
||||
const grouped = useMemo(() => {
|
||||
const q = search.trim().toLowerCase()
|
||||
const filtered = q
|
||||
? countries.filter(
|
||||
(c) => c.name.toLowerCase().includes(q) || c.code.toLowerCase().includes(q)
|
||||
)
|
||||
: countries
|
||||
|
||||
const buckets: Record<string, Country[]> = {}
|
||||
for (const country of filtered) {
|
||||
if (!buckets[country.continent]) buckets[country.continent] = []
|
||||
buckets[country.continent].push(country)
|
||||
}
|
||||
return Object.entries(buckets).sort(([a], [b]) => a.localeCompare(b))
|
||||
}, [countries, search])
|
||||
|
||||
const selectedCountries = useMemo(
|
||||
() => countries.filter((c) => selected.has(c.code)),
|
||||
[countries, selected]
|
||||
)
|
||||
|
||||
const installedCountrySet = useMemo(() => {
|
||||
const set = new Set<CountryCode>()
|
||||
for (const filename of installedFilenames) {
|
||||
const match = SINGLE_COUNTRY_FILENAME_RE.exec(filename)
|
||||
if (match) set.add(match[1].toUpperCase() as CountryCode)
|
||||
}
|
||||
return set
|
||||
}, [installedFilenames])
|
||||
|
||||
function toggleCountry(code: CountryCode) {
|
||||
setSelected((prev) => {
|
||||
const next = new Set(prev)
|
||||
if (next.has(code)) next.delete(code)
|
||||
else next.add(code)
|
||||
return next
|
||||
})
|
||||
}
|
||||
|
||||
function toggleGroup(group: CountryGroup) {
|
||||
setSelected((prev) => {
|
||||
const next = new Set(prev)
|
||||
const allIn = group.countries.every((c) => next.has(c))
|
||||
if (allIn) {
|
||||
group.countries.forEach((c) => next.delete(c))
|
||||
} else {
|
||||
group.countries.forEach((c) => next.add(c))
|
||||
}
|
||||
return next
|
||||
})
|
||||
}
|
||||
|
||||
function clearAll() {
|
||||
setSelected(new Set())
|
||||
}
|
||||
|
||||
// Auto-refresh the preflight whenever selection or maxzoom changes. Debounced
|
||||
// so rapid multi-select clicks and slider drags collapse into a single CDN
|
||||
// round-trip. Loading state only flips after the debounce expires so the UI
|
||||
// stays interactive during the wait. Stale-safe via requestId so an earlier
|
||||
// slow response can't clobber a later one.
|
||||
useEffect(() => {
|
||||
if (selected.size === 0) {
|
||||
setPreflight(null)
|
||||
setErrorMessage(null)
|
||||
setLoading(false)
|
||||
preflightRequestIdRef.current++
|
||||
return
|
||||
}
|
||||
|
||||
setErrorMessage(null)
|
||||
const timer = setTimeout(async () => {
|
||||
const requestId = ++preflightRequestIdRef.current
|
||||
setLoading(true)
|
||||
try {
|
||||
const res = await api.extractMapPreflight({
|
||||
countries: [...selected],
|
||||
maxzoom,
|
||||
})
|
||||
if (requestId !== preflightRequestIdRef.current) return
|
||||
if (!res) throw new Error('Preflight returned no data')
|
||||
setPreflight(res)
|
||||
} catch (err: any) {
|
||||
if (requestId !== preflightRequestIdRef.current) return
|
||||
console.error('Preflight failed:', err)
|
||||
setErrorMessage(err?.message ?? 'Estimate failed')
|
||||
} finally {
|
||||
if (requestId === preflightRequestIdRef.current) setLoading(false)
|
||||
}
|
||||
}, 1500)
|
||||
|
||||
return () => clearTimeout(timer)
|
||||
}, [selected, maxzoom])
|
||||
|
||||
async function startDownload() {
|
||||
if (selected.size === 0) {
|
||||
setErrorMessage('Pick at least one country before downloading.')
|
||||
return
|
||||
}
|
||||
if (loading || !preflight) {
|
||||
setErrorMessage('Still estimating size — hold on a moment.')
|
||||
return
|
||||
}
|
||||
try {
|
||||
setDownloading(true)
|
||||
setErrorMessage(null)
|
||||
await api.extractMapRegion({
|
||||
countries: [...selected],
|
||||
maxzoom,
|
||||
estimatedBytes: preflight?.bytes,
|
||||
})
|
||||
onDownloadStart?.()
|
||||
} catch (err: any) {
|
||||
console.error('Extract dispatch failed:', err)
|
||||
setErrorMessage(err?.message ?? 'Download failed')
|
||||
} finally {
|
||||
setDownloading(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<StyledModal
|
||||
{...modalProps}
|
||||
title="Download map by country or region"
|
||||
open={true}
|
||||
confirmText="Start Download"
|
||||
confirmIcon="IconDownload"
|
||||
cancelText="Cancel"
|
||||
confirmVariant="primary"
|
||||
confirmLoading={loading || downloading}
|
||||
cancelLoading={loading || downloading}
|
||||
onConfirm={startDownload}
|
||||
large
|
||||
>
|
||||
<div className="flex flex-col text-left gap-4 min-h-[60vh]">
|
||||
<div className="flex gap-3 items-stretch">
|
||||
<div className="relative flex-1">
|
||||
<IconSearch className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-text-muted" />
|
||||
<input
|
||||
type="text"
|
||||
value={search}
|
||||
onChange={(e) => setSearch(e.target.value)}
|
||||
placeholder={`Search ${countries.length} countries...`}
|
||||
className="w-full pl-9 pr-3 py-2 rounded-md border border-border-default bg-surface-primary text-text-primary text-sm focus:outline-none focus:ring-2 focus:ring-desert-green"
|
||||
/>
|
||||
</div>
|
||||
{selected.size > 0 && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={clearAll}
|
||||
className="text-sm text-text-muted hover:text-text-primary px-3 cursor-pointer"
|
||||
>
|
||||
Clear all
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{groups.length > 0 && (
|
||||
<div>
|
||||
<p className="text-xs uppercase tracking-wide text-text-muted mb-2">
|
||||
Quick picks
|
||||
</p>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{groups.map((group) => {
|
||||
const allIn =
|
||||
group.countries.length > 0 &&
|
||||
group.countries.every((c) => selected.has(c))
|
||||
return (
|
||||
<button
|
||||
key={group.id}
|
||||
type="button"
|
||||
onClick={() => toggleGroup(group)}
|
||||
className={classNames(
|
||||
'px-3 py-1.5 rounded-full text-xs font-medium border transition-colors cursor-pointer',
|
||||
allIn
|
||||
? 'bg-desert-green text-white border-desert-green'
|
||||
: 'bg-surface-primary text-text-primary border-border-default hover:border-desert-green'
|
||||
)}
|
||||
>
|
||||
{allIn && <IconCheck className="inline w-3 h-3 mr-1" />}
|
||||
{group.name}{' '}
|
||||
<span className="opacity-60">({group.countries.length})</span>
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex-1 overflow-y-auto max-h-96 border border-border-default rounded-md bg-surface-secondary">
|
||||
{countriesLoading ? (
|
||||
<div className="flex items-center justify-center h-40">
|
||||
<LoadingSpinner />
|
||||
</div>
|
||||
) : grouped.length === 0 ? (
|
||||
<p className="text-text-muted text-sm p-6 text-center">
|
||||
No countries match "{search}".
|
||||
</p>
|
||||
) : (
|
||||
grouped.map(([continent, list]) => (
|
||||
<div key={continent}>
|
||||
<div className="sticky top-0 bg-surface-secondary border-b border-border-default px-4 py-2 text-xs uppercase tracking-wide text-text-muted font-semibold z-10">
|
||||
{continent}
|
||||
</div>
|
||||
<ul>
|
||||
{list.map((country) => {
|
||||
const isSelected = selected.has(country.code)
|
||||
const isInstalled = installedCountrySet.has(country.code)
|
||||
return (
|
||||
<li key={country.code}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => toggleCountry(country.code)}
|
||||
className={classNames(
|
||||
'w-full flex items-center gap-3 px-4 py-2 text-left text-sm transition-colors cursor-pointer',
|
||||
isSelected
|
||||
? 'bg-desert-green/10 hover:bg-desert-green/15'
|
||||
: 'hover:bg-surface-primary'
|
||||
)}
|
||||
>
|
||||
<span
|
||||
className={classNames(
|
||||
'w-4 h-4 rounded border flex items-center justify-center shrink-0',
|
||||
isSelected
|
||||
? 'bg-desert-green border-desert-green'
|
||||
: 'border-border-default'
|
||||
)}
|
||||
>
|
||||
{isSelected && <IconCheck className="w-3 h-3 text-white" />}
|
||||
</span>
|
||||
<span className="flex-1 text-text-primary">{country.name}</span>
|
||||
{isInstalled && (
|
||||
<span
|
||||
className="text-[10px] uppercase tracking-wide font-semibold px-1.5 py-0.5 rounded bg-desert-green/15 text-desert-green border border-desert-green/30"
|
||||
title="Already downloaded — re-select to update with a different zoom"
|
||||
>
|
||||
Installed
|
||||
</span>
|
||||
)}
|
||||
<span className="text-xs font-mono text-text-muted">
|
||||
{country.code}
|
||||
</span>
|
||||
</button>
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
</ul>
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
|
||||
{selectedCountries.length > 0 && (
|
||||
<div>
|
||||
<p className="text-xs uppercase tracking-wide text-text-muted mb-2">
|
||||
{selectedCountries.length} selected
|
||||
</p>
|
||||
<div className="flex flex-wrap gap-2 max-h-24 overflow-y-auto">
|
||||
{selectedCountries.map((country) => (
|
||||
<span
|
||||
key={country.code}
|
||||
className="inline-flex items-center gap-1 px-2 py-1 rounded bg-desert-green text-white text-xs"
|
||||
>
|
||||
{country.name}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => toggleCountry(country.code)}
|
||||
className="hover:bg-white/20 rounded cursor-pointer"
|
||||
aria-label={`Remove ${country.name}`}
|
||||
>
|
||||
<IconX className="w-3 h-3" />
|
||||
</button>
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div>
|
||||
<label className="block text-sm text-text-primary font-medium mb-2">
|
||||
Max zoom level: <span className="font-mono">{maxzoom}</span>
|
||||
</label>
|
||||
<input
|
||||
type="range"
|
||||
min={EXTRACT_MIN_ZOOM}
|
||||
max={EXTRACT_MAX_ZOOM}
|
||||
step={1}
|
||||
value={maxzoom}
|
||||
onChange={(e) => setMaxzoom(parseInt(e.target.value, 10))}
|
||||
className="w-full accent-desert-green"
|
||||
disabled={downloading}
|
||||
/>
|
||||
<div className="flex justify-between text-xs text-text-muted mt-1 font-mono">
|
||||
<span>z{EXTRACT_MIN_ZOOM} (world)</span>
|
||||
<span>z{EXTRACT_MAX_ZOOM} (street)</span>
|
||||
</div>
|
||||
<p className="text-xs text-text-muted mt-2">
|
||||
Lower zoom = smaller file, less detail. Zoom 15 shows individual streets;
|
||||
zoom 10 shows city-level detail.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="bg-surface-secondary border border-border-default rounded-md p-3 min-h-14 text-sm font-mono">
|
||||
<PreflightStatus
|
||||
errorMessage={errorMessage}
|
||||
loading={loading}
|
||||
preflight={preflight}
|
||||
hasSelection={selected.size > 0}
|
||||
/>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</StyledModal>
|
||||
)
|
||||
}
|
||||
|
||||
type PreflightStatusProps = {
|
||||
errorMessage: string | null
|
||||
loading: boolean
|
||||
preflight: MapExtractPreflight | null
|
||||
hasSelection: boolean
|
||||
}
|
||||
|
||||
function PreflightStatus({ errorMessage, loading, preflight, hasSelection }: PreflightStatusProps) {
|
||||
if (errorMessage) {
|
||||
return <p className="text-desert-red">{errorMessage}</p>
|
||||
}
|
||||
if (loading) {
|
||||
return <p className="text-text-muted">Estimating size…</p>
|
||||
}
|
||||
if (preflight) {
|
||||
return (
|
||||
<p className="text-text-primary">
|
||||
{preflight.tiles.toLocaleString()} tiles, ~{formatBytes(preflight.bytes, 1)}{' '}
|
||||
<span className="text-text-muted">(source build {preflight.source.date})</span>
|
||||
</p>
|
||||
)
|
||||
}
|
||||
if (!hasSelection) {
|
||||
return <p className="text-text-muted">Pick at least one country to estimate size.</p>
|
||||
}
|
||||
return <p className="text-text-muted">Estimating size…</p>
|
||||
}
|
||||
|
||||
export default CountryPickerModal
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
type CoordinateOverlayProps = {
|
||||
latitude: number
|
||||
longitude: number
|
||||
x: number
|
||||
y: number
|
||||
}
|
||||
|
||||
export default function CoordinateOverlay({
|
||||
latitude,
|
||||
longitude,
|
||||
x,
|
||||
y,
|
||||
}: CoordinateOverlayProps) {
|
||||
return (
|
||||
<div
|
||||
className="pointer-events-none absolute z-[9999] -translate-x-1/2 whitespace-nowrap rounded bg-black/75 px-2 py-1 font-mono text-[11px] text-white"
|
||||
style={{
|
||||
left: x,
|
||||
top: y - 36,
|
||||
}}
|
||||
>
|
||||
{latitude.toFixed(6)}, {longitude.toFixed(6)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
@ -8,41 +8,66 @@ import Map, {
|
|||
import type { MapRef, MapLayerMouseEvent } from 'react-map-gl/maplibre'
|
||||
import maplibregl from 'maplibre-gl'
|
||||
import 'maplibre-gl/dist/maplibre-gl.css'
|
||||
|
||||
import { Protocol } from 'pmtiles'
|
||||
import { useEffect, useRef, useState, useCallback } from 'react'
|
||||
|
||||
import { useMapMarkers, PIN_COLORS } from '~/hooks/useMapMarkers'
|
||||
import MarkerPin from './MarkerPin'
|
||||
import MarkerPanel from './MarkerPanel'
|
||||
import CoordinateOverlay from './CoordinateOverlay'
|
||||
import ViewMapMarkerPopup from './ViewMapMarkerPopup'
|
||||
import MapMarkerFormPopup from './MapMarkerFormPopup'
|
||||
import ScaleUnitToggle from './ScaleUnitToggle'
|
||||
|
||||
type ScaleUnit = 'imperial' | 'metric'
|
||||
|
||||
export default function MapComponent() {
|
||||
const mapRef = useRef<MapRef>(null)
|
||||
const { markers, addMarker, updateMarker, deleteMarker } = useMapMarkers()
|
||||
type MapComponentProps = {
|
||||
isHoveringUI: boolean
|
||||
showCoordinatesEnabled: boolean
|
||||
}
|
||||
|
||||
export default function MapComponent({
|
||||
isHoveringUI,
|
||||
showCoordinatesEnabled,
|
||||
}: MapComponentProps) {
|
||||
const mapRef = useRef<MapRef>(null)
|
||||
const animationFrameRef = useRef<number | null>(null)
|
||||
const { markers, addMarker, updateMarker, deleteMarker } = useMapMarkers()
|
||||
const [isDraggingMap, setIsDraggingMap] = useState(false)
|
||||
const [placingMarker, setPlacingMarker] = useState<{ lng: number; lat: number } | null>(null)
|
||||
const [selectedMarkerId, setSelectedMarkerId] = useState<number | null>(null)
|
||||
const [editingMarkerId, setEditingMarkerId] = useState<number | null>(null)
|
||||
const [hasUnsavedMarkerChanges, setHasUnsavedMarkerChanges] = useState(false)
|
||||
|
||||
const hideCoordinates = useCallback(() => {
|
||||
setShowCoordinates(false)
|
||||
setCursorLngLat(null)
|
||||
}, [])
|
||||
|
||||
const [scaleUnit, setScaleUnit] = useState<ScaleUnit>(
|
||||
() => (localStorage.getItem('nomad:map-scale-unit') as ScaleUnit) || 'metric'
|
||||
)
|
||||
|
||||
const [cursorLngLat, setCursorLngLat] = useState<{
|
||||
lng: number
|
||||
lat: number
|
||||
x: number
|
||||
y: number
|
||||
} | null>(null)
|
||||
|
||||
const [showCoordinates, setShowCoordinates] = useState(false)
|
||||
|
||||
const confirmDiscardMarkerChanges = useCallback(() => {
|
||||
if (!hasUnsavedMarkerChanges) return true
|
||||
|
||||
return window.confirm('Discard unsaved marker changes?')
|
||||
}, [hasUnsavedMarkerChanges])
|
||||
|
||||
// Add the PMTiles protocol to maplibre-gl
|
||||
useEffect(() => {
|
||||
const protocol = new Protocol()
|
||||
maplibregl.addProtocol('pmtiles', protocol.tile)
|
||||
|
||||
return () => {
|
||||
maplibregl.removeProtocol('pmtiles')
|
||||
}
|
||||
|
|
@ -109,37 +134,52 @@ export default function MapComponent() {
|
|||
<ScaleControl position="bottom-left" maxWidth={150} unit={scaleUnit} />
|
||||
<ScaleUnitToggle scaleUnit={scaleUnit} onChange={handleScaleUnitChange} />
|
||||
|
||||
{markers.map((marker) => (
|
||||
<Marker
|
||||
key={marker.id}
|
||||
longitude={marker.longitude}
|
||||
latitude={marker.latitude}
|
||||
anchor="bottom"
|
||||
onClick={(e) => {
|
||||
e.originalEvent.stopPropagation()
|
||||
{markers
|
||||
.filter((marker) => marker.visible)
|
||||
.map((marker) => (
|
||||
<Marker
|
||||
key={marker.id}
|
||||
longitude={marker.longitude}
|
||||
latitude={marker.latitude}
|
||||
anchor="bottom"
|
||||
onClick={(e) => {
|
||||
e.originalEvent.stopPropagation()
|
||||
|
||||
if (!confirmDiscardMarkerChanges()) return
|
||||
if (!confirmDiscardMarkerChanges()) return
|
||||
|
||||
setSelectedMarkerId(marker.id === selectedMarkerId ? null : marker.id)
|
||||
setPlacingMarker(null)
|
||||
setEditingMarkerId(null)
|
||||
setHasUnsavedMarkerChanges(false)
|
||||
}}
|
||||
>
|
||||
<MarkerPin
|
||||
color={PIN_COLORS.find((color) => color.id === marker.color)?.hex}
|
||||
active={marker.id === selectedMarkerId}
|
||||
/>
|
||||
</Marker>
|
||||
))}
|
||||
setSelectedMarkerId(marker.id === selectedMarkerId ? null : marker.id)
|
||||
setPlacingMarker(null)
|
||||
setEditingMarkerId(null)
|
||||
setHasUnsavedMarkerChanges(false)
|
||||
}}
|
||||
>
|
||||
<MarkerPin
|
||||
color={marker.color}
|
||||
customColor={marker.customColor}
|
||||
icon={marker.icon}
|
||||
iconColor={marker.iconColor}
|
||||
visible={marker.visible}
|
||||
active={marker.id === selectedMarkerId}
|
||||
/>
|
||||
</Marker>
|
||||
))}
|
||||
|
||||
{placingMarker && (
|
||||
<MapMarkerFormPopup
|
||||
longitude={placingMarker.lng}
|
||||
latitude={placingMarker.lat}
|
||||
onDirtyChange={setHasUnsavedMarkerChanges}
|
||||
onSave={async ({ name, notes, color }) => {
|
||||
await addMarker(name, placingMarker.lng, placingMarker.lat, color, notes || undefined)
|
||||
onSave={async ({ name, notes, color, customColor, icon }) => {
|
||||
await addMarker({
|
||||
name,
|
||||
longitude: placingMarker.lng,
|
||||
latitude: placingMarker.lat,
|
||||
color,
|
||||
customColor,
|
||||
icon,
|
||||
notes: notes || null,
|
||||
})
|
||||
|
||||
setPlacingMarker(null)
|
||||
setHasUnsavedMarkerChanges(false)
|
||||
}}
|
||||
|
|
@ -167,13 +207,15 @@ export default function MapComponent() {
|
|||
latitude={selectedMarker.latitude}
|
||||
initialMarker={selectedMarker}
|
||||
onDirtyChange={setHasUnsavedMarkerChanges}
|
||||
onSave={async ({ id, name, notes, color }) => {
|
||||
onSave={async ({ id, name, notes, color, customColor, icon }) => {
|
||||
if (!id) return
|
||||
|
||||
await updateMarker(id, {
|
||||
name,
|
||||
notes: notes || null,
|
||||
color,
|
||||
customColor,
|
||||
icon,
|
||||
})
|
||||
|
||||
setEditingMarkerId(null)
|
||||
|
|
@ -184,13 +226,15 @@ export default function MapComponent() {
|
|||
)}
|
||||
</Map>
|
||||
|
||||
<MarkerPanel
|
||||
markers={markers}
|
||||
onDelete={handleDeleteMarker}
|
||||
onFlyTo={handleFlyTo}
|
||||
onSelect={setSelectedMarkerId}
|
||||
selectedMarkerId={selectedMarkerId}
|
||||
/>
|
||||
<div onMouseEnter={hideCoordinates}>
|
||||
<MarkerPanel
|
||||
markers={markers}
|
||||
onDelete={handleDeleteMarker}
|
||||
onFlyTo={handleFlyTo}
|
||||
onSelect={setSelectedMarkerId}
|
||||
selectedMarkerId={selectedMarkerId}
|
||||
/>
|
||||
</div>
|
||||
</MapProvider>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,35 +1,29 @@
|
|||
type ScaleUnit = 'imperial' | 'metric'
|
||||
|
||||
type ScaleUnitControlProps = {
|
||||
type ScaleUnitToggleProps = {
|
||||
scaleUnit: ScaleUnit
|
||||
onChange: (unit: ScaleUnit) => void
|
||||
onMouseEnter?: () => void
|
||||
}
|
||||
|
||||
export default function ScaleUnitToggle({ scaleUnit, onChange }: ScaleUnitControlProps) {
|
||||
export default function ScaleUnitToggle({
|
||||
scaleUnit,
|
||||
onChange,
|
||||
onMouseEnter,
|
||||
}: ScaleUnitToggleProps) {
|
||||
return (
|
||||
<div style={{ position: 'absolute', bottom: '30px', left: '10px', zIndex: 2 }}>
|
||||
<div
|
||||
style={{
|
||||
display: 'inline-flex',
|
||||
borderRadius: '4px',
|
||||
boxShadow: '0 0 0 2px rgba(0,0,0,0.1)',
|
||||
overflow: 'hidden',
|
||||
fontSize: '11px',
|
||||
fontWeight: 600,
|
||||
lineHeight: 1,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
className="absolute bottom-[30px] left-[10px] z-[2]"
|
||||
onMouseEnter={onMouseEnter}
|
||||
>
|
||||
<div className="inline-flex overflow-hidden rounded text-[11px] font-semibold leading-none shadow-[0_0_0_2px_rgba(0,0,0,0.1)]">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
if (scaleUnit !== 'metric') onChange('metric')
|
||||
}}
|
||||
onClick={() => onChange('metric')}
|
||||
className="border-0 px-2 py-1"
|
||||
style={{
|
||||
background: scaleUnit === 'metric' ? '#424420' : 'white',
|
||||
color: scaleUnit === 'metric' ? 'white' : '#666',
|
||||
border: 'none',
|
||||
padding: '4px 8px',
|
||||
cursor: 'pointer',
|
||||
}}
|
||||
>
|
||||
Metric
|
||||
|
|
@ -37,15 +31,11 @@ export default function ScaleUnitToggle({ scaleUnit, onChange }: ScaleUnitContro
|
|||
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
if (scaleUnit !== 'imperial') onChange('imperial')
|
||||
}}
|
||||
onClick={() => onChange('imperial')}
|
||||
className="border-0 px-2 py-1"
|
||||
style={{
|
||||
background: scaleUnit === 'imperial' ? '#424420' : 'white',
|
||||
color: scaleUnit === 'imperial' ? 'white' : '#666',
|
||||
border: 'none',
|
||||
padding: '4px 8px',
|
||||
cursor: 'pointer',
|
||||
}}
|
||||
>
|
||||
Imperial
|
||||
|
|
|
|||
|
|
@ -19,8 +19,9 @@ const useDownloads = (props: useDownloadsProps) => {
|
|||
queryFn: () => api.listDownloadJobs(props.filetype),
|
||||
refetchInterval: (query) => {
|
||||
const data = query.state.data
|
||||
// Only poll when there are active downloads; otherwise use a slower interval
|
||||
return data && data.length > 0 ? 2000 : 30000
|
||||
// Idle poll is kept tight so newly-dispatched jobs surface quickly — small ZIM
|
||||
// updates can complete in ~2s, so a 30s idle interval almost always missed them.
|
||||
return data && data.length > 0 ? 2000 : 3000
|
||||
},
|
||||
enabled: props.enabled ?? true,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
// eslint-disable-next-line @unicorn/filename-case
|
||||
import { useState, useCallback, useEffect } from 'react'
|
||||
import api from '~/lib/api'
|
||||
|
||||
|
|
@ -26,6 +27,7 @@ export function useMapMarkers() {
|
|||
const [markers, setMarkers] = useState<MapMarker[]>([])
|
||||
const [loaded, setLoaded] = useState(false)
|
||||
|
||||
// Load markers from API on mount
|
||||
useEffect(() => {
|
||||
api.listMapMarkers().then((data) => {
|
||||
if (data) {
|
||||
|
|
@ -98,11 +100,11 @@ export function useMapMarkers() {
|
|||
prev.map((m) =>
|
||||
m.id === id
|
||||
? {
|
||||
...m,
|
||||
name: result.name,
|
||||
color: result.color as PinColorId,
|
||||
notes: result.notes ?? null,
|
||||
}
|
||||
...m,
|
||||
name: result.name,
|
||||
color: result.color as PinColorId,
|
||||
notes: result.notes ?? null,
|
||||
}
|
||||
: m
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { ServiceSlim } from '../../types/services'
|
|||
import { FileEntry } from '../../types/files'
|
||||
import { CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus } from '../../types/system'
|
||||
import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads'
|
||||
import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps'
|
||||
import { EmbedJobWithProgress } from '../../types/rag'
|
||||
import type { CategoryWithStatus, CollectionWithStatus, ContentUpdateCheckResult, ResourceUpdateInfo } from '../../types/collections'
|
||||
import { catchInternal } from './util'
|
||||
|
|
@ -131,6 +132,15 @@ class API {
|
|||
})()
|
||||
}
|
||||
|
||||
async deleteMapRegionFile(filename: string): Promise<{ message: string }> {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.delete<{ message: string }>(
|
||||
`/maps/${encodeURIComponent(filename)}`
|
||||
)
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async downloadRemoteZimFile(
|
||||
url: string,
|
||||
metadata?: { title: string; summary?: string; author?: string; size_bytes?: number }
|
||||
|
|
@ -549,6 +559,46 @@ class API {
|
|||
})()
|
||||
}
|
||||
|
||||
async listCountries() {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.get<{ countries: Country[] }>('/maps/countries')
|
||||
return response.data.countries
|
||||
})()
|
||||
}
|
||||
|
||||
async listCountryGroups() {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.get<{ groups: CountryGroup[] }>('/maps/country-groups')
|
||||
return response.data.groups
|
||||
})()
|
||||
}
|
||||
|
||||
async extractMapPreflight(params: { countries: CountryCode[]; maxzoom?: number }) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.post<MapExtractPreflight>(
|
||||
'/maps/extract-preflight',
|
||||
params
|
||||
)
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async extractMapRegion(params: {
|
||||
countries: CountryCode[]
|
||||
maxzoom?: number
|
||||
label?: string
|
||||
estimatedBytes?: number
|
||||
}) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.post<{
|
||||
message: string
|
||||
filename: string
|
||||
jobId?: string
|
||||
}>('/maps/extract', params)
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async listCuratedMapCollections() {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.get<CollectionWithStatus[]>(
|
||||
|
|
@ -635,6 +685,42 @@ class API {
|
|||
})()
|
||||
}
|
||||
|
||||
async listCustomLibraries() {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.get<{ id: number; name: string; base_url: string; is_default: boolean }[]>(
|
||||
'/zim/custom-libraries'
|
||||
)
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async addCustomLibrary(name: string, base_url: string) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.post<{
|
||||
message: string
|
||||
library: { id: number; name: string; base_url: string }
|
||||
}>('/zim/custom-libraries', { name, base_url })
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async removeCustomLibrary(id: number) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.delete<{ message: string }>(`/zim/custom-libraries/${id}`)
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async browseLibrary(url: string) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.get<{
|
||||
directories: { name: string; url: string }[]
|
||||
files: { name: string; url: string; size_bytes: number | null }[]
|
||||
}>('/zim/browse-library', { params: { url } })
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async deleteZimFile(filename: string) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.delete<{ message: string }>(`/zim/${filename}`)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
export function hasDownloadedGlobalMap(
|
||||
globalMapKey: string | null | undefined,
|
||||
storedMapFiles: Array<{ name: string }>
|
||||
): boolean {
|
||||
if (!globalMapKey) {
|
||||
return false
|
||||
}
|
||||
|
||||
return storedMapFiles.some((file) => file.name === globalMapKey || /^\d{8}\.pmtiles$/.test(file.name))
|
||||
}
|
||||
|
|
@ -1,38 +1,66 @@
|
|||
import MapsLayout from '~/layouts/MapsLayout'
|
||||
import { useState } from 'react'
|
||||
import { Head, Link, router } from '@inertiajs/react'
|
||||
import { IconArrowLeft } from '@tabler/icons-react'
|
||||
|
||||
import MapsLayout from '~/layouts/MapsLayout'
|
||||
import MapComponent from '~/components/maps/MapComponent'
|
||||
import StyledButton from '~/components/StyledButton'
|
||||
import { IconArrowLeft } from '@tabler/icons-react'
|
||||
import { FileEntry } from '../../types/files'
|
||||
import Alert from '~/components/Alert'
|
||||
|
||||
import { FileEntry } from '../../types/files'
|
||||
|
||||
export default function Maps(props: {
|
||||
maps: { baseAssetsExist: boolean; regionFiles: FileEntry[] }
|
||||
}) {
|
||||
const [isHoveringUI, setIsHoveringUI] = useState(false)
|
||||
const [showMapCoordinates, setShowMapCoordinates] = useState(true)
|
||||
|
||||
const alertMessage = !props.maps.baseAssetsExist
|
||||
? 'The base map assets have not been installed. Please download them first to enable map functionality.'
|
||||
: props.maps.regionFiles.length === 0
|
||||
? 'No map regions have been downloaded yet. Please download some regions to enable map functionality.'
|
||||
: null
|
||||
? 'No map regions have been downloaded yet. Please download some regions to enable map functionality.'
|
||||
: null
|
||||
|
||||
return (
|
||||
<MapsLayout>
|
||||
<Head title="Maps" />
|
||||
|
||||
<div className="relative w-full h-screen overflow-hidden">
|
||||
{/* Nav and alerts are overlayed */}
|
||||
<div className="absolute top-0 left-0 right-0 z-50 flex justify-between p-4 bg-surface-secondary backdrop-blur-sm shadow-sm">
|
||||
{/* Navbar */}
|
||||
<div
|
||||
className="absolute top-0 left-0 right-0 z-50 flex justify-between p-4 bg-surface-secondary backdrop-blur-sm shadow-sm"
|
||||
onMouseEnter={() => setIsHoveringUI(true)}
|
||||
onMouseLeave={() => setIsHoveringUI(false)}
|
||||
>
|
||||
<Link href="/home" className="flex items-center">
|
||||
<IconArrowLeft className="mr-2" size={24} />
|
||||
<p className="text-lg text-text-secondary">Back to Home</p>
|
||||
</Link>
|
||||
<Link href="/settings/maps" className='mr-4'>
|
||||
<StyledButton variant="primary" icon="IconSettings">
|
||||
Manage Map Regions
|
||||
</StyledButton>
|
||||
</Link>
|
||||
|
||||
<div className="flex items-center gap-3 mr-4">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setShowMapCoordinates((prev) => !prev)}
|
||||
className="rounded px-3 py-2 text-sm bg-surface-primary text-text-secondary hover:opacity-80 transition"
|
||||
>
|
||||
{showMapCoordinates ? 'Hide Coordinates' : 'Show Coordinates'}
|
||||
</button>
|
||||
|
||||
<Link href="/settings/maps">
|
||||
<StyledButton variant="primary" icon="IconSettings">
|
||||
Manage Map Regions
|
||||
</StyledButton>
|
||||
</Link>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Alert */}
|
||||
{alertMessage && (
|
||||
<div className="absolute top-20 left-4 right-4 z-50">
|
||||
<div
|
||||
className="absolute top-20 left-4 right-4 z-50"
|
||||
onMouseEnter={() => setIsHoveringUI(true)}
|
||||
onMouseLeave={() => setIsHoveringUI(false)}
|
||||
>
|
||||
<Alert
|
||||
title={alertMessage}
|
||||
type="warning"
|
||||
|
|
@ -47,8 +75,13 @@ export default function Maps(props: {
|
|||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Map */}
|
||||
<div className="absolute inset-0">
|
||||
<MapComponent />
|
||||
<MapComponent
|
||||
isHoveringUI={isHoveringUI}
|
||||
showCoordinatesEnabled={showMapCoordinates}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</MapsLayout>
|
||||
|
|
|
|||
|
|
@ -6,17 +6,19 @@ import { useModals } from '~/context/ModalContext'
|
|||
import StyledModal from '~/components/StyledModal'
|
||||
import { FileEntry } from '../../../types/files'
|
||||
import { useNotifications } from '~/context/NotificationContext'
|
||||
import { useState } from 'react'
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import api from '~/lib/api'
|
||||
import DownloadURLModal from '~/components/DownloadURLModal'
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import useDownloads from '~/hooks/useDownloads'
|
||||
import StyledSectionHeader from '~/components/StyledSectionHeader'
|
||||
import CuratedCollectionCard from '~/components/CuratedCollectionCard'
|
||||
import CountryPickerModal from '~/components/CountryPickerModal'
|
||||
import type { CollectionWithStatus } from '../../../types/collections'
|
||||
import ActiveDownloads from '~/components/ActiveDownloads'
|
||||
import Alert from '~/components/Alert'
|
||||
import { formatBytes } from '~/lib/util'
|
||||
import { hasDownloadedGlobalMap } from '~/lib/global_map_banner'
|
||||
|
||||
const CURATED_COLLECTIONS_KEY = 'curated-map-collections'
|
||||
const GLOBAL_MAP_INFO_KEY = 'global-map-info'
|
||||
|
|
@ -28,6 +30,7 @@ export default function MapsManager(props: {
|
|||
const { openModal, closeAllModals } = useModals()
|
||||
const { addNotification } = useNotifications()
|
||||
const [downloading, setDownloading] = useState(false)
|
||||
const [deletingFileKey, setDeletingFileKey] = useState<string | null>(null)
|
||||
|
||||
const { data: curatedCollections } = useQuery({
|
||||
queryKey: [CURATED_COLLECTIONS_KEY],
|
||||
|
|
@ -35,16 +38,28 @@ export default function MapsManager(props: {
|
|||
refetchOnWindowFocus: false,
|
||||
})
|
||||
|
||||
const { invalidate: invalidateDownloads } = useDownloads({
|
||||
const { data: activeMapDownloads = [], invalidate: invalidateDownloads } = useDownloads({
|
||||
filetype: 'map',
|
||||
enabled: true,
|
||||
})
|
||||
|
||||
// Refresh the Stored Map Files list when a map download finishes. We pass props.maps.regionFiles
|
||||
// straight through from the server-side render, so without an Inertia partial reload it stays stale
|
||||
// until the user navigates away and back.
|
||||
const prevMapDownloadCountRef = useRef(activeMapDownloads.length)
|
||||
useEffect(() => {
|
||||
if (activeMapDownloads.length < prevMapDownloadCountRef.current) {
|
||||
router.reload({ only: ['maps'] })
|
||||
}
|
||||
prevMapDownloadCountRef.current = activeMapDownloads.length
|
||||
}, [activeMapDownloads.length])
|
||||
|
||||
const { data: globalMapInfo } = useQuery({
|
||||
queryKey: [GLOBAL_MAP_INFO_KEY],
|
||||
queryFn: () => api.getGlobalMapInfo(),
|
||||
refetchOnWindowFocus: false,
|
||||
})
|
||||
const globalMapAlreadyDownloaded = hasDownloadedGlobalMap(globalMapInfo?.key, props.maps.regionFiles)
|
||||
|
||||
const downloadGlobalMap = useMutation({
|
||||
mutationFn: () => api.downloadGlobalMap(),
|
||||
|
|
@ -118,18 +133,40 @@ export default function MapsManager(props: {
|
|||
}
|
||||
}
|
||||
|
||||
async function deleteFile(file: FileEntry) {
|
||||
if (file.type !== 'file') return
|
||||
|
||||
try {
|
||||
setDeletingFileKey(file.key)
|
||||
await api.deleteMapRegionFile(file.name)
|
||||
addNotification({
|
||||
type: 'success',
|
||||
message: `${file.name} has been deleted.`,
|
||||
})
|
||||
closeAllModals()
|
||||
router.reload({ only: ['maps'] })
|
||||
} catch (error) {
|
||||
console.error('Error deleting map file:', error)
|
||||
addNotification({
|
||||
type: 'error',
|
||||
message: `Failed to delete ${file.name}. Please try again.`,
|
||||
})
|
||||
} finally {
|
||||
setDeletingFileKey(null)
|
||||
}
|
||||
}
|
||||
|
||||
async function confirmDeleteFile(file: FileEntry) {
|
||||
openModal(
|
||||
<StyledModal
|
||||
title="Confirm Delete?"
|
||||
onConfirm={() => {
|
||||
closeAllModals()
|
||||
}}
|
||||
onConfirm={() => deleteFile(file)}
|
||||
onCancel={closeAllModals}
|
||||
open={true}
|
||||
confirmText="Delete"
|
||||
cancelText="Cancel"
|
||||
confirmVariant="danger"
|
||||
confirmLoading={file.type === 'file' && deletingFileKey === file.key}
|
||||
>
|
||||
<p className="text-text-secondary">
|
||||
Are you sure you want to delete {file.name}? This action cannot be undone.
|
||||
|
|
@ -196,6 +233,24 @@ export default function MapsManager(props: {
|
|||
)
|
||||
}
|
||||
|
||||
function openCountryPickerModal() {
|
||||
openModal(
|
||||
<CountryPickerModal
|
||||
onCancel={closeAllModals}
|
||||
installedFilenames={(props.maps.regionFiles ?? []).map((f) => f.name)}
|
||||
onDownloadStart={() => {
|
||||
invalidateDownloads()
|
||||
addNotification({
|
||||
type: 'success',
|
||||
message: 'Download queued. Watch progress below.',
|
||||
})
|
||||
closeAllModals()
|
||||
}}
|
||||
/>,
|
||||
'country-picker-modal'
|
||||
)
|
||||
}
|
||||
|
||||
async function openDownloadModal() {
|
||||
openModal(
|
||||
<DownloadURLModal
|
||||
|
|
@ -251,7 +306,23 @@ export default function MapsManager(props: {
|
|||
}}
|
||||
/>
|
||||
)}
|
||||
{globalMapInfo && (
|
||||
{globalMapInfo && globalMapAlreadyDownloaded && (
|
||||
<Alert
|
||||
title="Global Map Installed"
|
||||
message={`Your global map build ${globalMapInfo.date} (${formatBytes(globalMapInfo.size, 1)}) is stored locally and ready for offline use.`}
|
||||
type="success"
|
||||
variant="bordered"
|
||||
className="mt-8"
|
||||
icon="IconCircleCheck"
|
||||
buttonProps={{
|
||||
variant: 'secondary',
|
||||
children: 'Download latest build',
|
||||
icon: 'IconRefresh',
|
||||
onClick: () => confirmGlobalMapDownload(),
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
{globalMapInfo && !globalMapAlreadyDownloaded && (
|
||||
<Alert
|
||||
title="Global Map Coverage Available"
|
||||
message={`Download a complete worldwide map from Protomaps (${formatBytes(globalMapInfo.size, 1)}, build ${globalMapInfo.date}). This is a large file but covers the entire planet — no individual region downloads needed.`}
|
||||
|
|
@ -268,6 +339,21 @@ export default function MapsManager(props: {
|
|||
}}
|
||||
/>
|
||||
)}
|
||||
<Alert
|
||||
title="Download by country or region"
|
||||
message="Pick the countries you actually need — from a single country to a whole continent — and we'll pull just those tiles from the global Protomaps archive. Much smaller than the full 125 GB global map."
|
||||
type="info-inverted"
|
||||
variant="bordered"
|
||||
className="mt-8"
|
||||
icon="IconMap2"
|
||||
buttonProps={{
|
||||
variant: 'primary',
|
||||
children: 'Choose Countries',
|
||||
icon: 'IconMap2',
|
||||
onClick: openCountryPickerModal,
|
||||
}}
|
||||
/>
|
||||
|
||||
<div className="mt-8 mb-6 flex items-center justify-between">
|
||||
<StyledSectionHeader title="Curated Map Regions" className="!mb-0" />
|
||||
<StyledButton
|
||||
|
|
|
|||
|
|
@ -283,7 +283,7 @@ export default function ModelsPage(props: {
|
|||
type="warning"
|
||||
variant="bordered"
|
||||
title="GPU Not Accessible"
|
||||
message={`Your system has an NVIDIA GPU, but ${aiAssistantName} can't access it. AI is running on CPU only, which is significantly slower.`}
|
||||
message={`Your system has ${systemInfo?.gpuHealth?.gpuVendor === 'amd' ? 'an AMD' : 'an NVIDIA'} GPU, but ${aiAssistantName} can't access it. AI is running on CPU only, which is significantly slower.`}
|
||||
className="!mt-6"
|
||||
dismissible={true}
|
||||
onDismiss={handleDismissGpuBanner}
|
||||
|
|
|
|||
|
|
@ -209,7 +209,7 @@ export default function SettingsPage(props: {
|
|||
type="warning"
|
||||
variant="bordered"
|
||||
title="GPU Not Accessible to AI Assistant"
|
||||
message="Your system has an NVIDIA GPU, but the AI Assistant can't access it. AI is running on CPU only, which is significantly slower."
|
||||
message={`Your system has ${info?.gpuHealth?.gpuVendor === 'amd' ? 'an AMD' : 'an NVIDIA'} GPU, but the AI Assistant can't access it. AI is running on CPU only, which is significantly slower.`}
|
||||
dismissible={true}
|
||||
onDismiss={handleDismissGpuBanner}
|
||||
buttonProps={{
|
||||
|
|
|
|||
|
|
@ -5,16 +5,17 @@ import StyledTable from '~/components/StyledTable'
|
|||
import StyledSectionHeader from '~/components/StyledSectionHeader'
|
||||
import ActiveDownloads from '~/components/ActiveDownloads'
|
||||
import Alert from '~/components/Alert'
|
||||
import { useEffect, useState } from 'react'
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import { IconAlertCircle, IconArrowBigUpLines, IconCheck, IconCircleCheck, IconReload } from '@tabler/icons-react'
|
||||
import { SystemUpdateStatus } from '../../../types/system'
|
||||
import type { ContentUpdateCheckResult, ResourceUpdateInfo } from '../../../types/collections'
|
||||
import api from '~/lib/api'
|
||||
import Input from '~/components/inputs/Input'
|
||||
import Switch from '~/components/inputs/Switch'
|
||||
import { useMutation } from '@tanstack/react-query'
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import { useNotifications } from '~/context/NotificationContext'
|
||||
import { useSystemSetting } from '~/hooks/useSystemSetting'
|
||||
import { formatBytes } from '~/lib/util'
|
||||
|
||||
type Props = {
|
||||
updateAvailable: boolean
|
||||
|
|
@ -23,8 +24,26 @@ type Props = {
|
|||
earlyAccess: boolean
|
||||
}
|
||||
|
||||
const STAGE_LABELS: Record<SystemUpdateStatus['stage'], string> = {
|
||||
idle: 'Preparing Update',
|
||||
starting: 'Starting Update',
|
||||
pulling: 'Pulling Images',
|
||||
pulled: 'Images Pulled',
|
||||
recreating: 'Recreating Containers',
|
||||
complete: 'Update Complete',
|
||||
error: 'Update Failed',
|
||||
}
|
||||
|
||||
const ADVANCED_STAGES: ReadonlySet<SystemUpdateStatus['stage']> = new Set([
|
||||
'pulling',
|
||||
'pulled',
|
||||
'recreating',
|
||||
'complete',
|
||||
])
|
||||
|
||||
function ContentUpdatesSection() {
|
||||
const { addNotification } = useNotifications()
|
||||
const queryClient = useQueryClient()
|
||||
const [checkResult, setCheckResult] = useState<ContentUpdateCheckResult | null>(null)
|
||||
const [isChecking, setIsChecking] = useState(false)
|
||||
const [applyingIds, setApplyingIds] = useState<Set<string>>(new Set())
|
||||
|
|
@ -60,6 +79,9 @@ function ContentUpdatesSection() {
|
|||
? { ...prev, updates: prev.updates.filter((u) => u.resource_id !== update.resource_id) }
|
||||
: prev
|
||||
)
|
||||
// Force Active Downloads to refetch now — small updates finish before the next
|
||||
// idle poll fires, so without this the user wouldn't see them.
|
||||
queryClient.invalidateQueries({ queryKey: ['download-jobs'] })
|
||||
} else {
|
||||
addNotification({ type: 'error', message: result?.error || 'Failed to start update' })
|
||||
}
|
||||
|
|
@ -95,6 +117,9 @@ function ContentUpdatesSection() {
|
|||
? { ...prev, updates: prev.updates.filter((u) => !successIds.has(u.resource_id)) }
|
||||
: prev
|
||||
)
|
||||
if (successIds.size > 0) {
|
||||
queryClient.invalidateQueries({ queryKey: ['download-jobs'] })
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
addNotification({ type: 'error', message: 'Failed to apply updates' })
|
||||
|
|
@ -182,6 +207,15 @@ function ContentUpdatesSection() {
|
|||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
accessor: 'size_bytes',
|
||||
title: 'Size',
|
||||
render: (record) => (
|
||||
<span className="text-desert-stone-dark">
|
||||
{record.size_bytes ? formatBytes(record.size_bytes, 1) : '—'}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
accessor: 'installed_version',
|
||||
title: 'Version',
|
||||
|
|
@ -234,6 +268,12 @@ export default function SystemUpdatePage(props: { system: Props }) {
|
|||
const [email, setEmail] = useState('')
|
||||
const [versionInfo, setVersionInfo] = useState<Omit<Props, 'earlyAccess'>>(props.system)
|
||||
const [showConnectionLostNotice, setShowConnectionLostNotice] = useState(false)
|
||||
// Tracks whether this update session has progressed past 'idle'/'starting'.
|
||||
// The sidecar sits on 'complete' for ~5s before resetting to 'idle' (see
|
||||
// install/sidecar-updater/update-watcher.sh), and the SPA can miss that
|
||||
// window across the admin container restart. If we resurface to 'idle'
|
||||
// after seeing an advanced stage, treat it as the missed completion.
|
||||
const seenAdvancedStageRef = useRef(false)
|
||||
|
||||
const earlyAccessSetting = useSystemSetting({
|
||||
key: 'system.earlyAccess', initialData: {
|
||||
|
|
@ -253,11 +293,22 @@ export default function SystemUpdatePage(props: { system: Props }) {
|
|||
}
|
||||
setUpdateStatus(response)
|
||||
|
||||
if (ADVANCED_STAGES.has(response.stage)) {
|
||||
seenAdvancedStageRef.current = true
|
||||
}
|
||||
|
||||
// If we can connect again, hide the connection lost notice
|
||||
setShowConnectionLostNotice(false)
|
||||
|
||||
// Check if update is complete or errored
|
||||
if (response.stage === 'complete') {
|
||||
// Check if update is complete or errored. We also treat a return to
|
||||
// 'idle' as completion if we previously saw an advanced stage — this
|
||||
// catches the race where the sidecar's brief 'complete' window passes
|
||||
// while we're disconnected during the admin container restart.
|
||||
const isComplete =
|
||||
response.stage === 'complete' ||
|
||||
(response.stage === 'idle' && seenAdvancedStageRef.current)
|
||||
|
||||
if (isComplete) {
|
||||
// Re-check version so the KV store clears the stale "update available" flag
|
||||
// before we reload, otherwise the banner shows "current → current"
|
||||
try {
|
||||
|
|
@ -287,6 +338,7 @@ export default function SystemUpdatePage(props: { system: Props }) {
|
|||
const handleStartUpdate = async () => {
|
||||
try {
|
||||
setError(null)
|
||||
seenAdvancedStageRef.current = false
|
||||
setIsUpdating(true)
|
||||
const response = await api.startSystemUpdate()
|
||||
if (!response || !response.success) {
|
||||
|
|
@ -351,7 +403,7 @@ export default function SystemUpdatePage(props: { system: Props }) {
|
|||
if (updateStatus?.stage === 'error')
|
||||
return <IconAlertCircle className="h-12 w-12 text-desert-red" />
|
||||
if (isUpdating) return <IconReload className="h-12 w-12 text-desert-green animate-spin" />
|
||||
if (props.system.updateAvailable)
|
||||
if (versionInfo.updateAvailable)
|
||||
return <IconArrowBigUpLines className="h-16 w-16 text-desert-green" />
|
||||
return <IconCircleCheck className="h-16 w-16 text-desert-olive" />
|
||||
}
|
||||
|
|
@ -363,6 +415,9 @@ export default function SystemUpdatePage(props: { system: Props }) {
|
|||
onSuccess: () => {
|
||||
addNotification({ message: 'Setting updated successfully.', type: 'success' })
|
||||
earlyAccessSetting.refetch()
|
||||
// Toggling Early Access changes which versions are eligible, so re-evaluate
|
||||
// immediately rather than making the user click Check Again.
|
||||
checkVersionMutation.mutate()
|
||||
},
|
||||
onError: (error) => {
|
||||
console.error('Error updating setting:', error)
|
||||
|
|
@ -444,11 +499,11 @@ export default function SystemUpdatePage(props: { system: Props }) {
|
|||
{!isUpdating && (
|
||||
<>
|
||||
<h2 className="text-2xl font-bold text-desert-green mb-2">
|
||||
{props.system.updateAvailable ? 'Update Available' : 'System Up to Date'}
|
||||
{versionInfo.updateAvailable ? 'Update Available' : 'System Up to Date'}
|
||||
</h2>
|
||||
<p className="text-desert-stone-dark mb-6">
|
||||
{props.system.updateAvailable
|
||||
? `A new version (${props.system.latestVersion}) is available for your Project N.O.M.A.D. instance.`
|
||||
{versionInfo.updateAvailable
|
||||
? `A new version (${versionInfo.latestVersion}) is available for your Project N.O.M.A.D. instance.`
|
||||
: 'Your system is running the latest version!'}
|
||||
</p>
|
||||
</>
|
||||
|
|
@ -456,8 +511,8 @@ export default function SystemUpdatePage(props: { system: Props }) {
|
|||
|
||||
{isUpdating && updateStatus && (
|
||||
<>
|
||||
<h2 className="text-2xl font-bold text-desert-green mb-2 capitalize">
|
||||
{updateStatus.stage === 'idle' ? 'Preparing Update' : updateStatus.stage}
|
||||
<h2 className="text-2xl font-bold text-desert-green mb-2">
|
||||
{STAGE_LABELS[updateStatus.stage] ?? updateStatus.stage}
|
||||
</h2>
|
||||
<p className="text-desert-stone-dark mb-6">{updateStatus.message}</p>
|
||||
</>
|
||||
|
|
|
|||
|
|
@ -21,7 +21,16 @@ import useInternetStatus from '~/hooks/useInternetStatus'
|
|||
import Alert from '~/components/Alert'
|
||||
import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
|
||||
import Input from '~/components/inputs/Input'
|
||||
import { IconSearch, IconBooks } from '@tabler/icons-react'
|
||||
import {
|
||||
IconSearch,
|
||||
IconBooks,
|
||||
IconFolder,
|
||||
IconFileDownload,
|
||||
IconChevronRight,
|
||||
IconPlus,
|
||||
IconTrash,
|
||||
IconLibrary,
|
||||
} from '@tabler/icons-react'
|
||||
import useDebounce from '~/hooks/useDebounce'
|
||||
import CategoryCard from '~/components/CategoryCard'
|
||||
import TierSelectionModal from '~/components/TierSelectionModal'
|
||||
|
|
@ -34,6 +43,13 @@ import { SERVICE_NAMES } from '../../../../constants/service_names'
|
|||
|
||||
const CURATED_CATEGORIES_KEY = 'curated-categories'
|
||||
const WIKIPEDIA_STATE_KEY = 'wikipedia-state'
|
||||
const CUSTOM_LIBRARIES_KEY = 'custom-libraries'
|
||||
|
||||
type CustomLibrary = { id: number; name: string; base_url: string; is_default: boolean }
|
||||
type BrowseResult = {
|
||||
directories: { name: string; url: string }[]
|
||||
files: { name: string; url: string; size_bytes: number | null }[]
|
||||
}
|
||||
|
||||
export default function ZimRemoteExplorer() {
|
||||
const queryClient = useQueryClient()
|
||||
|
|
@ -56,6 +72,20 @@ export default function ZimRemoteExplorer() {
|
|||
const [selectedWikipedia, setSelectedWikipedia] = useState<string | null>(null)
|
||||
const [isSubmittingWikipedia, setIsSubmittingWikipedia] = useState(false)
|
||||
|
||||
// Custom library state - persist selection to localStorage
|
||||
const [selectedSource, setSelectedSource] = useState<'default' | number>(() => {
|
||||
try {
|
||||
const saved = localStorage.getItem('nomad:zim-library-source')
|
||||
if (saved && saved !== 'default') return parseInt(saved, 10)
|
||||
} catch {}
|
||||
return 'default'
|
||||
})
|
||||
const [browseUrl, setBrowseUrl] = useState<string | null>(null)
|
||||
const [breadcrumbs, setBreadcrumbs] = useState<{ name: string; url: string }[]>([])
|
||||
const [manageModalOpen, setManageModalOpen] = useState(false)
|
||||
const [newLibraryName, setNewLibraryName] = useState('')
|
||||
const [newLibraryUrl, setNewLibraryUrl] = useState('')
|
||||
|
||||
const debouncedSetQuery = debounce((val: string) => {
|
||||
setQuery(val)
|
||||
}, 400)
|
||||
|
|
@ -79,6 +109,26 @@ export default function ZimRemoteExplorer() {
|
|||
enabled: true,
|
||||
})
|
||||
|
||||
// Fetch custom libraries
|
||||
const { data: customLibraries } = useQuery({
|
||||
queryKey: [CUSTOM_LIBRARIES_KEY],
|
||||
queryFn: () => api.listCustomLibraries(),
|
||||
refetchOnWindowFocus: false,
|
||||
})
|
||||
|
||||
// Browse custom library directory
|
||||
const {
|
||||
data: browseData,
|
||||
isLoading: isBrowsing,
|
||||
error: browseError,
|
||||
} = useQuery<BrowseResult>({
|
||||
queryKey: ['browse-library', browseUrl],
|
||||
queryFn: () => api.browseLibrary(browseUrl!) as Promise<BrowseResult>,
|
||||
enabled: !!browseUrl && selectedSource !== 'default',
|
||||
refetchOnWindowFocus: false,
|
||||
retry: false,
|
||||
})
|
||||
|
||||
const { data, fetchNextPage, isFetching, isLoading } =
|
||||
useInfiniteQuery<ListRemoteZimFilesResponse>({
|
||||
queryKey: ['remote-zim-files', query],
|
||||
|
|
@ -97,6 +147,7 @@ export default function ZimRemoteExplorer() {
|
|||
getNextPageParam: (lastPage) => (lastPage.has_more ? lastPage.next_start : undefined),
|
||||
refetchOnWindowFocus: false,
|
||||
placeholderData: keepPreviousData,
|
||||
enabled: selectedSource === 'default',
|
||||
})
|
||||
|
||||
const flatData = useMemo(() => {
|
||||
|
|
@ -140,6 +191,50 @@ export default function ZimRemoteExplorer() {
|
|||
fetchOnBottomReached(tableParentRef.current)
|
||||
}, [fetchOnBottomReached])
|
||||
|
||||
// Restore custom library selection on mount when data loads
|
||||
useEffect(() => {
|
||||
if (selectedSource !== 'default' && customLibraries) {
|
||||
const lib = customLibraries.find((l) => l.id === selectedSource)
|
||||
if (lib && !browseUrl) {
|
||||
setBrowseUrl(lib.base_url)
|
||||
setBreadcrumbs([{ name: lib.name, url: lib.base_url }])
|
||||
} else if (!lib) {
|
||||
// Saved library was deleted
|
||||
setSelectedSource('default')
|
||||
localStorage.setItem('nomad:zim-library-source', 'default')
|
||||
}
|
||||
}
|
||||
}, [customLibraries, selectedSource])
|
||||
|
||||
// When selecting a custom library, navigate to its root
|
||||
const handleSourceChange = (value: string) => {
|
||||
localStorage.setItem('nomad:zim-library-source', value)
|
||||
if (value === 'default') {
|
||||
setSelectedSource('default')
|
||||
setBrowseUrl(null)
|
||||
setBreadcrumbs([])
|
||||
} else {
|
||||
const id = parseInt(value, 10)
|
||||
const lib = customLibraries?.find((l) => l.id === id)
|
||||
if (lib) {
|
||||
setSelectedSource(id)
|
||||
setBrowseUrl(lib.base_url)
|
||||
setBreadcrumbs([{ name: lib.name, url: lib.base_url }])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const navigateToDirectory = (name: string, url: string) => {
|
||||
setBrowseUrl(url)
|
||||
setBreadcrumbs((prev) => [...prev, { name, url }])
|
||||
}
|
||||
|
||||
const navigateToBreadcrumb = (index: number) => {
|
||||
const crumb = breadcrumbs[index]
|
||||
setBrowseUrl(crumb.url)
|
||||
setBreadcrumbs((prev) => prev.slice(0, index + 1))
|
||||
}
|
||||
|
||||
async function confirmDownload(record: RemoteZimFileEntry) {
|
||||
openModal(
|
||||
<StyledModal
|
||||
|
|
@ -165,6 +260,31 @@ export default function ZimRemoteExplorer() {
|
|||
)
|
||||
}
|
||||
|
||||
async function confirmCustomDownload(file: { name: string; url: string; size_bytes: number | null }) {
|
||||
openModal(
|
||||
<StyledModal
|
||||
title="Confirm Download?"
|
||||
onConfirm={() => {
|
||||
downloadCustomFile(file)
|
||||
closeAllModals()
|
||||
}}
|
||||
onCancel={closeAllModals}
|
||||
open={true}
|
||||
confirmText="Download"
|
||||
cancelText="Cancel"
|
||||
confirmVariant="primary"
|
||||
>
|
||||
<p className="text-text-primary">
|
||||
Are you sure you want to download{' '}
|
||||
<strong>{file.name}</strong>
|
||||
{file.size_bytes ? ` (${formatBytes(file.size_bytes)})` : ''}? The Kiwix
|
||||
application will be restarted after the download is complete.
|
||||
</p>
|
||||
</StyledModal>,
|
||||
'confirm-download-custom-modal'
|
||||
)
|
||||
}
|
||||
|
||||
async function downloadFile(record: RemoteZimFileEntry) {
|
||||
try {
|
||||
await api.downloadRemoteZimFile(record.download_url, {
|
||||
|
|
@ -179,6 +299,26 @@ export default function ZimRemoteExplorer() {
|
|||
}
|
||||
}
|
||||
|
||||
async function downloadCustomFile(file: { name: string; url: string; size_bytes: number | null }) {
|
||||
try {
|
||||
await api.downloadRemoteZimFile(file.url, {
|
||||
title: file.name.replace(/\.zim$/, ''),
|
||||
size_bytes: file.size_bytes ?? undefined,
|
||||
})
|
||||
addNotification({
|
||||
message: `Started downloading "${file.name}"`,
|
||||
type: 'success',
|
||||
})
|
||||
invalidateDownloads()
|
||||
} catch (error) {
|
||||
console.error('Error downloading file:', error)
|
||||
addNotification({
|
||||
message: 'Failed to start download.',
|
||||
type: 'error',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Category/tier handlers
|
||||
const handleCategoryClick = (category: CategoryWithStatus) => {
|
||||
if (!isOnline) return
|
||||
|
|
@ -264,6 +404,35 @@ export default function ZimRemoteExplorer() {
|
|||
},
|
||||
})
|
||||
|
||||
// Custom library management
|
||||
const addLibraryMutation = useMutation({
|
||||
mutationFn: () => api.addCustomLibrary(newLibraryName.trim(), newLibraryUrl.trim()),
|
||||
onSuccess: () => {
|
||||
addNotification({ message: 'Custom library added.', type: 'success' })
|
||||
queryClient.invalidateQueries({ queryKey: [CUSTOM_LIBRARIES_KEY] })
|
||||
setNewLibraryName('')
|
||||
setNewLibraryUrl('')
|
||||
},
|
||||
onError: () => {
|
||||
addNotification({ message: 'Failed to add custom library.', type: 'error' })
|
||||
},
|
||||
})
|
||||
|
||||
const removeLibraryMutation = useMutation({
|
||||
mutationFn: (id: number) => api.removeCustomLibrary(id),
|
||||
onSuccess: (_data, id) => {
|
||||
addNotification({ message: 'Custom library removed.', type: 'success' })
|
||||
queryClient.invalidateQueries({ queryKey: [CUSTOM_LIBRARIES_KEY] })
|
||||
if (selectedSource === id) {
|
||||
setSelectedSource('default')
|
||||
setBrowseUrl(null)
|
||||
setBreadcrumbs([])
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
const hasCustomLibraries = customLibraries && customLibraries.length > 0
|
||||
|
||||
return (
|
||||
<SettingsLayout>
|
||||
<Head title="Content Explorer | Project N.O.M.A.D." />
|
||||
|
|
@ -302,7 +471,7 @@ export default function ZimRemoteExplorer() {
|
|||
Force Refresh Collections
|
||||
</StyledButton>
|
||||
</div>
|
||||
|
||||
|
||||
{/* Wikipedia Selector */}
|
||||
{isLoadingWikipedia ? (
|
||||
<div className="mt-8 bg-surface-primary rounded-lg border border-border-subtle p-6">
|
||||
|
|
@ -360,87 +529,303 @@ export default function ZimRemoteExplorer() {
|
|||
) : (
|
||||
<p className="text-text-muted mt-4">No curated content categories available.</p>
|
||||
)}
|
||||
<StyledSectionHeader title="Browse the Kiwix Library" className="mt-12 mb-4" />
|
||||
<div className="flex justify-start mt-4">
|
||||
<Input
|
||||
name="search"
|
||||
label=""
|
||||
placeholder="Search available ZIM files..."
|
||||
value={queryUI}
|
||||
onChange={(e) => {
|
||||
setQueryUI(e.target.value)
|
||||
debouncedSetQuery(e.target.value)
|
||||
}}
|
||||
className="w-1/3"
|
||||
leftIcon={<IconSearch className="w-5 h-5 text-text-muted" />}
|
||||
/>
|
||||
|
||||
{/* Kiwix Library / Custom Library Browser */}
|
||||
<div className="mt-12 mb-4 flex items-center justify-between">
|
||||
<StyledSectionHeader title="Browse the Kiwix Library" className="!mb-0" />
|
||||
<StyledButton
|
||||
onClick={() => setManageModalOpen(true)}
|
||||
disabled={!isOnline}
|
||||
icon="IconLibrary"
|
||||
>
|
||||
{hasCustomLibraries ? 'Manage Custom Libraries' : 'Add Custom Library'}
|
||||
</StyledButton>
|
||||
</div>
|
||||
<StyledTable<RemoteZimFileEntry & { actions?: any }>
|
||||
data={flatData.map((i, idx) => {
|
||||
const row = virtualizer.getVirtualItems().find((v) => v.index === idx)
|
||||
return {
|
||||
...i,
|
||||
height: `${row?.size || 48}px`, // Use the size from the virtualizer
|
||||
translateY: row?.start || 0,
|
||||
}
|
||||
})}
|
||||
ref={tableParentRef}
|
||||
loading={isLoading}
|
||||
columns={[
|
||||
{
|
||||
accessor: 'title',
|
||||
},
|
||||
{
|
||||
accessor: 'author',
|
||||
},
|
||||
{
|
||||
accessor: 'summary',
|
||||
},
|
||||
{
|
||||
accessor: 'updated',
|
||||
render(record) {
|
||||
return new Intl.DateTimeFormat('en-US', {
|
||||
dateStyle: 'medium',
|
||||
}).format(new Date(record.updated))
|
||||
},
|
||||
},
|
||||
{
|
||||
accessor: 'size_bytes',
|
||||
title: 'Size',
|
||||
render(record) {
|
||||
return formatBytes(record.size_bytes)
|
||||
},
|
||||
},
|
||||
{
|
||||
accessor: 'actions',
|
||||
render(record) {
|
||||
return (
|
||||
<div className="flex space-x-2">
|
||||
<StyledButton
|
||||
icon={'IconDownload'}
|
||||
onClick={() => {
|
||||
confirmDownload(record)
|
||||
}}
|
||||
|
||||
{/* Source selector dropdown */}
|
||||
{hasCustomLibraries && (
|
||||
<div className="flex items-center gap-3 mb-4">
|
||||
<label className="text-sm font-medium text-text-secondary">Source:</label>
|
||||
<select
|
||||
value={selectedSource === 'default' ? 'default' : String(selectedSource)}
|
||||
onChange={(e) => handleSourceChange(e.target.value)}
|
||||
className="rounded-md border border-border-default bg-surface-primary text-text-primary px-3 py-1.5 text-sm focus:outline-none focus:ring-2 focus:ring-desert-green"
|
||||
>
|
||||
<option value="default">Default (Kiwix)</option>
|
||||
{customLibraries.map((lib) => (
|
||||
<option key={lib.id} value={String(lib.id)}>
|
||||
{lib.name}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Default Kiwix library browser */}
|
||||
{selectedSource === 'default' && (
|
||||
<>
|
||||
<div className="flex justify-start mt-4">
|
||||
<Input
|
||||
name="search"
|
||||
label=""
|
||||
placeholder="Search available ZIM files..."
|
||||
value={queryUI}
|
||||
onChange={(e) => {
|
||||
setQueryUI(e.target.value)
|
||||
debouncedSetQuery(e.target.value)
|
||||
}}
|
||||
className="w-1/3"
|
||||
leftIcon={<IconSearch className="w-5 h-5 text-text-muted" />}
|
||||
/>
|
||||
</div>
|
||||
<StyledTable<RemoteZimFileEntry & { actions?: any }>
|
||||
data={flatData.map((i, idx) => {
|
||||
const row = virtualizer.getVirtualItems().find((v) => v.index === idx)
|
||||
return {
|
||||
...i,
|
||||
height: `${row?.size || 48}px`,
|
||||
translateY: row?.start || 0,
|
||||
}
|
||||
})}
|
||||
ref={tableParentRef}
|
||||
loading={isLoading}
|
||||
columns={[
|
||||
{
|
||||
accessor: 'title',
|
||||
},
|
||||
{
|
||||
accessor: 'author',
|
||||
},
|
||||
{
|
||||
accessor: 'summary',
|
||||
},
|
||||
{
|
||||
accessor: 'updated',
|
||||
render(record) {
|
||||
return new Intl.DateTimeFormat('en-US', {
|
||||
dateStyle: 'medium',
|
||||
}).format(new Date(record.updated))
|
||||
},
|
||||
},
|
||||
{
|
||||
accessor: 'size_bytes',
|
||||
title: 'Size',
|
||||
render(record) {
|
||||
return formatBytes(record.size_bytes)
|
||||
},
|
||||
},
|
||||
{
|
||||
accessor: 'actions',
|
||||
render(record) {
|
||||
return (
|
||||
<div className="flex space-x-2">
|
||||
<StyledButton
|
||||
icon={'IconDownload'}
|
||||
onClick={() => {
|
||||
confirmDownload(record)
|
||||
}}
|
||||
>
|
||||
Download
|
||||
</StyledButton>
|
||||
</div>
|
||||
)
|
||||
},
|
||||
},
|
||||
]}
|
||||
className="relative overflow-x-auto overflow-y-auto h-[600px] w-full mt-4"
|
||||
tableBodyStyle={{
|
||||
position: 'relative',
|
||||
height: `${virtualizer.getTotalSize()}px`,
|
||||
}}
|
||||
containerProps={{
|
||||
onScroll: (e) => fetchOnBottomReached(e.currentTarget as HTMLDivElement),
|
||||
}}
|
||||
compact
|
||||
rowLines
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Custom library directory browser */}
|
||||
{selectedSource !== 'default' && (
|
||||
<div className="mt-4">
|
||||
{/* Breadcrumb navigation */}
|
||||
<nav className="flex items-center gap-1 text-sm text-text-muted mb-4 flex-wrap">
|
||||
{breadcrumbs.map((crumb, idx) => (
|
||||
<span key={idx} className="flex items-center gap-1">
|
||||
{idx > 0 && <IconChevronRight className="w-4 h-4" />}
|
||||
{idx < breadcrumbs.length - 1 ? (
|
||||
<button
|
||||
onClick={() => navigateToBreadcrumb(idx)}
|
||||
className="text-desert-green hover:underline"
|
||||
>
|
||||
Download
|
||||
</StyledButton>
|
||||
</div>
|
||||
)
|
||||
},
|
||||
},
|
||||
]}
|
||||
className="relative overflow-x-auto overflow-y-auto h-[600px] w-full mt-4"
|
||||
tableBodyStyle={{
|
||||
position: 'relative',
|
||||
height: `${virtualizer.getTotalSize()}px`,
|
||||
}}
|
||||
containerProps={{
|
||||
onScroll: (e) => fetchOnBottomReached(e.currentTarget as HTMLDivElement),
|
||||
}}
|
||||
compact
|
||||
rowLines
|
||||
/>
|
||||
{crumb.name}
|
||||
</button>
|
||||
) : (
|
||||
<span className="text-text-primary font-medium">{crumb.name}</span>
|
||||
)}
|
||||
</span>
|
||||
))}
|
||||
</nav>
|
||||
|
||||
{isBrowsing && (
|
||||
<div className="flex justify-center py-12">
|
||||
<div className="animate-spin rounded-full h-8 w-8 border-b-2 border-desert-green"></div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{browseError && (
|
||||
<Alert
|
||||
title="Could not fetch directory listing from this URL."
|
||||
message="The server may not support directory browsing, or the URL may be incorrect."
|
||||
type="error"
|
||||
variant="solid"
|
||||
/>
|
||||
)}
|
||||
|
||||
{!isBrowsing && !browseError && browseData && (
|
||||
<div className="bg-surface-primary rounded-lg border border-border-subtle overflow-hidden relative" style={{ maxHeight: '600px', overflowY: 'auto' }}>
|
||||
{browseData.directories.length === 0 && browseData.files.length === 0 ? (
|
||||
<p className="text-text-muted p-6 text-center">
|
||||
No directories or ZIM files found at this location.
|
||||
</p>
|
||||
) : (
|
||||
<table className="w-full text-sm">
|
||||
<thead>
|
||||
<tr className="border-b border-border-subtle bg-surface-secondary sticky top-0 z-10">
|
||||
<th className="text-left px-4 py-3 font-medium text-text-secondary">Name</th>
|
||||
<th className="text-right px-4 py-3 font-medium text-text-secondary w-32">Size</th>
|
||||
<th className="text-right px-4 py-3 font-medium text-text-secondary w-36"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{browseData.directories.map((dir) => (
|
||||
<tr
|
||||
key={dir.url}
|
||||
className="border-b border-border-subtle hover:bg-surface-secondary cursor-pointer transition-colors"
|
||||
onClick={() => navigateToDirectory(dir.name, dir.url)}
|
||||
>
|
||||
<td className="px-4 py-3">
|
||||
<span className="flex items-center gap-2 text-text-primary">
|
||||
<IconFolder className="w-5 h-5 text-desert-orange" />
|
||||
{dir.name}
|
||||
</span>
|
||||
</td>
|
||||
<td className="text-right px-4 py-3 text-text-muted">--</td>
|
||||
<td className="text-right px-4 py-3">
|
||||
<IconChevronRight className="w-4 h-4 text-text-muted ml-auto" />
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
{browseData.files.map((file) => (
|
||||
<tr
|
||||
key={file.url}
|
||||
className="border-b border-border-subtle hover:bg-surface-secondary transition-colors"
|
||||
>
|
||||
<td className="px-4 py-3">
|
||||
<span className="flex items-center gap-2 text-text-primary">
|
||||
<IconFileDownload className="w-5 h-5 text-desert-green" />
|
||||
{file.name}
|
||||
</span>
|
||||
</td>
|
||||
<td className="text-right px-4 py-3 text-text-muted">
|
||||
{file.size_bytes ? formatBytes(file.size_bytes) : '--'}
|
||||
</td>
|
||||
<td className="text-right px-4 py-3">
|
||||
<StyledButton
|
||||
icon="IconDownload"
|
||||
onClick={() => confirmCustomDownload(file)}
|
||||
>
|
||||
Download
|
||||
</StyledButton>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<ActiveDownloads filetype="zim" withHeader />
|
||||
|
||||
{/* Manage Custom Libraries Modal */}
|
||||
<StyledModal
|
||||
title="Manage Custom Libraries"
|
||||
open={manageModalOpen}
|
||||
onCancel={() => setManageModalOpen(false)}
|
||||
cancelText="Close"
|
||||
>
|
||||
<div className="space-y-6">
|
||||
<div>
|
||||
<p className="text-sm text-text-muted mb-4">
|
||||
Add Kiwix mirrors or other ZIM file sources for faster downloads.
|
||||
</p>
|
||||
|
||||
{/* Existing libraries */}
|
||||
{customLibraries && customLibraries.length > 0 && (
|
||||
<div className="space-y-2 mb-6">
|
||||
{customLibraries.map((lib) => (
|
||||
<div
|
||||
key={lib.id}
|
||||
className="flex items-center justify-between bg-surface-secondary rounded-lg px-4 py-3 border border-border-subtle"
|
||||
>
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="font-medium text-text-primary truncate">
|
||||
{lib.name}
|
||||
{lib.is_default && (
|
||||
<span className="ml-2 text-xs text-text-muted font-normal">(built-in)</span>
|
||||
)}
|
||||
</p>
|
||||
<p className="text-xs text-text-muted truncate">{lib.base_url}</p>
|
||||
</div>
|
||||
{!lib.is_default && (
|
||||
<button
|
||||
onClick={() => removeLibraryMutation.mutate(lib.id)}
|
||||
className="ml-3 p-1.5 text-text-muted hover:text-red-500 transition-colors rounded"
|
||||
title="Remove library"
|
||||
>
|
||||
<IconTrash className="w-4 h-4" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Add new library form */}
|
||||
<div className="space-y-3">
|
||||
<Input
|
||||
name="library-name"
|
||||
label="Library Name"
|
||||
placeholder="e.g., Debian Mirror"
|
||||
value={newLibraryName}
|
||||
onChange={(e) => setNewLibraryName(e.target.value)}
|
||||
/>
|
||||
<Input
|
||||
name="library-url"
|
||||
label="Base URL"
|
||||
placeholder="e.g., https://cdimage.debian.org/mirror/kiwix.org/zim/"
|
||||
value={newLibraryUrl}
|
||||
onChange={(e) => setNewLibraryUrl(e.target.value)}
|
||||
/>
|
||||
<StyledButton
|
||||
icon="IconPlus"
|
||||
onClick={() => addLibraryMutation.mutate()}
|
||||
disabled={
|
||||
!newLibraryName.trim() ||
|
||||
!newLibraryUrl.trim() ||
|
||||
addLibraryMutation.isPending
|
||||
}
|
||||
>
|
||||
Add Library
|
||||
</StyledButton>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</StyledModal>
|
||||
</main>
|
||||
</div>
|
||||
</SettingsLayout>
|
||||
|
|
|
|||
|
|
@ -12184,9 +12184,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/micromatch/node_modules/picomatch": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
|
||||
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
|
||||
"version": "2.3.2",
|
||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz",
|
||||
"integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8.6"
|
||||
|
|
@ -13361,9 +13361,9 @@
|
|||
"license": "ISC"
|
||||
},
|
||||
"node_modules/picomatch": {
|
||||
"version": "4.0.3",
|
||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
|
||||
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz",
|
||||
"integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
|
|
|
|||
|
|
@ -38,94 +38,94 @@
|
|||
"#jobs/*": "./app/jobs/*.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@adonisjs/assembler": "^7.8.2",
|
||||
"@adonisjs/eslint-config": "^2.0.0",
|
||||
"@adonisjs/prettier-config": "^1.4.4",
|
||||
"@adonisjs/tsconfig": "^1.4.0",
|
||||
"@japa/assert": "^4.0.1",
|
||||
"@japa/plugin-adonisjs": "^4.0.0",
|
||||
"@japa/runner": "^4.2.0",
|
||||
"@adonisjs/assembler": "7.8.2",
|
||||
"@adonisjs/eslint-config": "2.1.2",
|
||||
"@adonisjs/prettier-config": "1.4.5",
|
||||
"@adonisjs/tsconfig": "1.4.1",
|
||||
"@japa/assert": "4.2.0",
|
||||
"@japa/plugin-adonisjs": "4.0.0",
|
||||
"@japa/runner": "4.5.0",
|
||||
"@swc/core": "1.11.24",
|
||||
"@tanstack/eslint-plugin-query": "^5.81.2",
|
||||
"@types/compression": "^1.8.1",
|
||||
"@types/dockerode": "^4.0.1",
|
||||
"@types/luxon": "^3.6.2",
|
||||
"@types/node": "^22.15.18",
|
||||
"@types/react": "^19.1.8",
|
||||
"@types/react-dom": "^19.1.6",
|
||||
"@types/stopword": "^2.0.3",
|
||||
"eslint": "^9.26.0",
|
||||
"hot-hook": "^0.4.0",
|
||||
"prettier": "^3.5.3",
|
||||
"ts-node-maintained": "^10.9.5",
|
||||
"typescript": "~5.8.3",
|
||||
"vite": "^6.4.2"
|
||||
"@tanstack/eslint-plugin-query": "5.91.4",
|
||||
"@types/compression": "1.8.1",
|
||||
"@types/dockerode": "4.0.1",
|
||||
"@types/luxon": "3.7.1",
|
||||
"@types/node": "22.19.7",
|
||||
"@types/react": "19.2.10",
|
||||
"@types/react-dom": "19.2.3",
|
||||
"@types/stopword": "2.0.3",
|
||||
"eslint": "9.39.2",
|
||||
"hot-hook": "0.4.0",
|
||||
"prettier": "3.8.1",
|
||||
"ts-node-maintained": "10.9.6",
|
||||
"typescript": "5.8.3",
|
||||
"vite": "6.4.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"@adonisjs/auth": "^9.4.0",
|
||||
"@adonisjs/core": "^6.18.0",
|
||||
"@adonisjs/cors": "^2.2.1",
|
||||
"@adonisjs/inertia": "^3.1.1",
|
||||
"@adonisjs/lucid": "^21.8.2",
|
||||
"@adonisjs/session": "^7.5.1",
|
||||
"@adonisjs/shield": "^8.2.0",
|
||||
"@adonisjs/static": "^1.1.1",
|
||||
"@adonisjs/transmit": "^2.0.2",
|
||||
"@adonisjs/transmit-client": "^1.0.0",
|
||||
"@adonisjs/vite": "^4.0.0",
|
||||
"@chonkiejs/core": "^0.0.7",
|
||||
"@headlessui/react": "^2.2.4",
|
||||
"@inertiajs/react": "^2.0.13",
|
||||
"@markdoc/markdoc": "^0.5.2",
|
||||
"@openzim/libzim": "^4.0.0",
|
||||
"@protomaps/basemaps": "^5.7.0",
|
||||
"@qdrant/js-client-rest": "^1.16.2",
|
||||
"@tabler/icons-react": "^3.34.0",
|
||||
"@tailwindcss/vite": "^4.1.10",
|
||||
"@tanstack/react-query": "^5.81.5",
|
||||
"@tanstack/react-query-devtools": "^5.83.0",
|
||||
"@tanstack/react-virtual": "^3.13.12",
|
||||
"@uppy/core": "^5.2.0",
|
||||
"@uppy/dashboard": "^5.1.0",
|
||||
"@uppy/react": "^5.1.1",
|
||||
"@vinejs/vine": "^3.0.1",
|
||||
"@vitejs/plugin-react": "^4.6.0",
|
||||
"autoprefixer": "^10.4.21",
|
||||
"axios": "^1.15.0",
|
||||
"better-sqlite3": "^12.1.1",
|
||||
"bullmq": "^5.65.1",
|
||||
"cheerio": "^1.2.0",
|
||||
"compression": "^1.8.1",
|
||||
"dockerode": "^4.0.7",
|
||||
"edge.js": "^6.2.1",
|
||||
"fast-xml-parser": "^5.5.7",
|
||||
"fuse.js": "^7.1.0",
|
||||
"jszip": "^3.10.1",
|
||||
"luxon": "^3.6.1",
|
||||
"maplibre-gl": "^4.7.1",
|
||||
"mysql2": "^3.14.1",
|
||||
"ollama": "^0.6.3",
|
||||
"openai": "^6.27.0",
|
||||
"pdf-parse": "^2.4.5",
|
||||
"pdf2pic": "^3.2.0",
|
||||
"pino-pretty": "^13.0.0",
|
||||
"pmtiles": "^4.4.0",
|
||||
"postcss": "^8.5.6",
|
||||
"react": "^19.1.0",
|
||||
"react-adonis-transmit": "^1.0.1",
|
||||
"react-dom": "^19.1.0",
|
||||
"react-map-gl": "^8.1.0",
|
||||
"react-markdown": "^10.1.0",
|
||||
"reflect-metadata": "^0.2.2",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"sharp": "^0.34.5",
|
||||
"stopword": "^3.1.5",
|
||||
"systeminformation": "^5.31.0",
|
||||
"tailwindcss": "^4.2.1",
|
||||
"tar": "^7.5.11",
|
||||
"tesseract.js": "^7.0.0",
|
||||
"url-join": "^5.0.0",
|
||||
"yaml": "^2.8.3"
|
||||
"@adonisjs/auth": "9.6.0",
|
||||
"@adonisjs/core": "6.19.3",
|
||||
"@adonisjs/cors": "2.2.1",
|
||||
"@adonisjs/inertia": "3.1.1",
|
||||
"@adonisjs/lucid": "21.8.2",
|
||||
"@adonisjs/session": "7.7.1",
|
||||
"@adonisjs/shield": "8.2.0",
|
||||
"@adonisjs/static": "1.1.1",
|
||||
"@adonisjs/transmit": "2.0.2",
|
||||
"@adonisjs/transmit-client": "1.1.0",
|
||||
"@adonisjs/vite": "4.0.0",
|
||||
"@chonkiejs/core": "0.0.7",
|
||||
"@headlessui/react": "2.2.9",
|
||||
"@inertiajs/react": "2.3.13",
|
||||
"@markdoc/markdoc": "0.5.4",
|
||||
"@openzim/libzim": "4.0.0",
|
||||
"@protomaps/basemaps": "5.7.0",
|
||||
"@qdrant/js-client-rest": "1.16.2",
|
||||
"@tabler/icons-react": "3.36.1",
|
||||
"@tailwindcss/vite": "4.1.18",
|
||||
"@tanstack/react-query": "5.90.20",
|
||||
"@tanstack/react-query-devtools": "5.91.3",
|
||||
"@tanstack/react-virtual": "3.13.18",
|
||||
"@uppy/core": "5.2.0",
|
||||
"@uppy/dashboard": "5.1.0",
|
||||
"@uppy/react": "5.1.1",
|
||||
"@vinejs/vine": "3.0.1",
|
||||
"@vitejs/plugin-react": "4.7.0",
|
||||
"autoprefixer": "10.4.24",
|
||||
"axios": "1.15.0",
|
||||
"better-sqlite3": "12.6.2",
|
||||
"bullmq": "5.67.2",
|
||||
"cheerio": "1.2.0",
|
||||
"compression": "1.8.1",
|
||||
"dockerode": "4.0.9",
|
||||
"edge.js": "6.4.0",
|
||||
"fast-xml-parser": "5.5.9",
|
||||
"fuse.js": "7.1.0",
|
||||
"jszip": "3.10.1",
|
||||
"luxon": "3.7.2",
|
||||
"maplibre-gl": "4.7.1",
|
||||
"mysql2": "3.16.2",
|
||||
"ollama": "0.6.3",
|
||||
"openai": "6.27.0",
|
||||
"pdf-parse": "2.4.5",
|
||||
"pdf2pic": "3.2.0",
|
||||
"pino-pretty": "13.1.3",
|
||||
"pmtiles": "4.4.0",
|
||||
"postcss": "8.5.6",
|
||||
"react": "19.2.4",
|
||||
"react-adonis-transmit": "1.0.1",
|
||||
"react-dom": "19.2.4",
|
||||
"react-map-gl": "8.1.0",
|
||||
"react-markdown": "10.1.0",
|
||||
"reflect-metadata": "0.2.2",
|
||||
"remark-gfm": "4.0.1",
|
||||
"sharp": "0.34.5",
|
||||
"stopword": "3.1.5",
|
||||
"systeminformation": "5.31.0",
|
||||
"tailwindcss": "4.2.2",
|
||||
"tar": "7.5.11",
|
||||
"tesseract.js": "7.0.0",
|
||||
"url-join": "5.0.0",
|
||||
"yaml": "2.8.3"
|
||||
},
|
||||
"hotHook": {
|
||||
"boundaries": [
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
import logger from '@adonisjs/core/services/logger'
|
||||
import type { ApplicationService } from '@adonisjs/core/types'
|
||||
|
||||
/**
|
||||
* Self-heals stale `system.updateAvailable` after a sidecar-driven update.
|
||||
*
|
||||
* When the admin container is recreated on a new image, the KVStore still
|
||||
* carries pre-update values for `system.updateAvailable` and
|
||||
* `system.latestVersion`. Without intervention the UI keeps showing the
|
||||
* "update available" banner until the next scheduled CheckUpdateJob (could be up to ~12h).
|
||||
*
|
||||
* Synchronous self-heal (no network): if the cached "latest" is not newer
|
||||
* than the version we are now running, clear `updateAvailable`. The next
|
||||
* scheduled CheckUpdateJob refreshes the cache from GitHub — we deliberately
|
||||
* do not hit the network from boot to avoid coupling container startup to
|
||||
* a network request to Github (e.g. container restart loop = flooding GitHub with requests).
|
||||
*
|
||||
* Note: this provider does not set `updateAvailable` to true if the cached
|
||||
* "latest" is newer than the current version. We rely on the next scheduled
|
||||
* CheckUpdateJob to do that, to avoid false positives in case of a stale cache.
|
||||
*/
|
||||
export default class VersionCheckProvider {
|
||||
constructor(protected app: ApplicationService) { }
|
||||
|
||||
async boot() {
|
||||
if (this.app.getEnvironment() !== 'web') return
|
||||
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
const KVStore = (await import('#models/kv_store')).default
|
||||
const { SystemService } = await import('#services/system_service')
|
||||
const { isNewerVersion } = await import('../app/utils/version.js')
|
||||
|
||||
const current = SystemService.getAppVersion()
|
||||
if (current === 'dev' || current === '0.0.0'){
|
||||
logger.info(`[VersionCheckProvider] Skipping self-heal for version ${current}. Appears to be a dev build without proper version set.`)
|
||||
return
|
||||
}
|
||||
|
||||
logger.info(`[VersionCheckProvider] Checking for stale updateAvailable (current=${current})`)
|
||||
|
||||
const cachedLatest = (await KVStore.getValue('system.latestVersion')) as string | null
|
||||
const earlyAccess = ((await KVStore.getValue('system.earlyAccess')) ?? false) as boolean
|
||||
|
||||
if (cachedLatest && !isNewerVersion(cachedLatest, current, earlyAccess)) {
|
||||
await KVStore.setValue('system.updateAvailable', false)
|
||||
logger.info(
|
||||
`[VersionCheckProvider] Cleared stale updateAvailable (cached=${cachedLatest}, current=${current})`
|
||||
)
|
||||
}
|
||||
} catch (err: any) {
|
||||
logger.warn(`[VersionCheckProvider] Self-heal skipped: ${err?.message ?? err}`)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -80,6 +80,10 @@ router
|
|||
router.post('/download-collection', [MapsController, 'downloadCollection'])
|
||||
router.get('/global-map-info', [MapsController, 'globalMapInfo'])
|
||||
router.post('/download-global-map', [MapsController, 'downloadGlobalMap'])
|
||||
router.get('/countries', [MapsController, 'listCountries'])
|
||||
router.get('/country-groups', [MapsController, 'listCountryGroups'])
|
||||
router.post('/extract-preflight', [MapsController, 'extractPreflight'])
|
||||
router.post('/extract', [MapsController, 'extractRegion'])
|
||||
router.get('/markers', [MapsController, 'listMarkers'])
|
||||
router.post('/markers', [MapsController, 'createMarker'])
|
||||
router.patch('/markers/:id', [MapsController, 'updateMarker'])
|
||||
|
|
@ -179,6 +183,12 @@ router
|
|||
|
||||
router.get('/wikipedia', [ZimController, 'getWikipediaState'])
|
||||
router.post('/wikipedia/select', [ZimController, 'selectWikipedia'])
|
||||
|
||||
router.get('/custom-libraries', [ZimController, 'listCustomLibraries'])
|
||||
router.post('/custom-libraries', [ZimController, 'addCustomLibrary'])
|
||||
router.delete('/custom-libraries/:id', [ZimController, 'removeCustomLibrary'])
|
||||
router.get('/browse-library', [ZimController, 'browseLibrary'])
|
||||
|
||||
router.delete('/:filename', [ZimController, 'delete'])
|
||||
})
|
||||
.prefix('/api/zim')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
import * as assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
|
||||
import { hasDownloadedGlobalMap } from '../../inertia/lib/global_map_banner.js'
|
||||
|
||||
test('returns true when the global map key already exists on disk', () => {
|
||||
assert.equal(
|
||||
hasDownloadedGlobalMap('20260402.pmtiles', [
|
||||
{ name: '20260402.pmtiles' },
|
||||
{ name: 'california.pmtiles' },
|
||||
]),
|
||||
true
|
||||
)
|
||||
})
|
||||
|
||||
test('returns false when the global map key is missing', () => {
|
||||
assert.equal(
|
||||
hasDownloadedGlobalMap('20260402.pmtiles', [
|
||||
{ name: 'california.pmtiles' },
|
||||
]),
|
||||
false
|
||||
)
|
||||
})
|
||||
|
||||
test('returns true when an older global map build is already on disk', () => {
|
||||
assert.equal(
|
||||
hasDownloadedGlobalMap('20260402.pmtiles', [
|
||||
{ name: '20260315.pmtiles' },
|
||||
{ name: 'california.pmtiles' },
|
||||
]),
|
||||
true
|
||||
)
|
||||
})
|
||||
|
||||
test('returns false when there is no global map info', () => {
|
||||
assert.equal(hasDownloadedGlobalMap(undefined, [{ name: '20260402.pmtiles' }]), false)
|
||||
})
|
||||
|
|
@ -86,6 +86,7 @@ export type ResourceUpdateInfo = {
|
|||
installed_version: string
|
||||
latest_version: string
|
||||
download_url: string
|
||||
size_bytes?: number
|
||||
}
|
||||
|
||||
export type ContentUpdateCheckResult = {
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ export const KV_STORE_SCHEMA = {
|
|||
'gpu.type': 'string',
|
||||
'ai.remoteOllamaUrl': 'string',
|
||||
'ai.ollamaFlashAttention': 'boolean',
|
||||
'ai.amdGpuAcceleration': 'boolean',
|
||||
} as const
|
||||
|
||||
type KVTagToType<T extends string> = T extends 'boolean' ? boolean : string
|
||||
|
|
|
|||
|
|
@ -22,6 +22,40 @@ export type MapLayer = {
|
|||
[key: string]: any
|
||||
}
|
||||
|
||||
/** ISO 3166-1 alpha-2 country code (e.g. "DE", "FR", "US"). */
|
||||
export type CountryCode = string
|
||||
|
||||
export type Country = {
|
||||
code: CountryCode
|
||||
code3: string
|
||||
name: string
|
||||
continent: string
|
||||
subregion: string
|
||||
population: number
|
||||
}
|
||||
|
||||
export type CountryGroup = {
|
||||
id: string
|
||||
name: string
|
||||
description: string
|
||||
countries: CountryCode[]
|
||||
}
|
||||
|
||||
export type MapExtractRequest = {
|
||||
countries: CountryCode[]
|
||||
maxzoom?: number
|
||||
}
|
||||
|
||||
export type MapExtractPreflight = {
|
||||
tiles: number
|
||||
bytes: number
|
||||
source: {
|
||||
url: string
|
||||
date: string
|
||||
key: string
|
||||
}
|
||||
}
|
||||
|
||||
export type MapMarkerResponse = {
|
||||
id: number
|
||||
name: string
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ import { Systeminformation } from 'systeminformation'
|
|||
export type GpuHealthStatus = {
|
||||
status: 'ok' | 'passthrough_failed' | 'no_gpu' | 'ollama_not_installed'
|
||||
hasNvidiaRuntime: boolean
|
||||
hasRocmRuntime: boolean
|
||||
ollamaGpuAccessible: boolean
|
||||
gpuVendor?: 'nvidia' | 'amd'
|
||||
}
|
||||
|
||||
export type SystemInformationResponse = {
|
||||
|
|
|
|||
|
|
@ -517,18 +517,35 @@ verify_gpu_setup() {
|
|||
echo -e "${YELLOW}○${RESET} Docker NVIDIA runtime not detected\\n"
|
||||
fi
|
||||
|
||||
# Check for AMD GPU
|
||||
# Check for AMD GPU — restrict to display controller classes to avoid false positives
|
||||
# from AMD CPU host bridges, PCI bridges, and chipset devices.
|
||||
local has_amd_gpu='false'
|
||||
if command -v lspci &> /dev/null; then
|
||||
if lspci 2>/dev/null | grep -iE "amd|radeon" &> /dev/null; then
|
||||
echo -e "${YELLOW}○${RESET} AMD GPU detected (ROCm support not currently available)\\n"
|
||||
if lspci 2>/dev/null | grep -iE "VGA|3D controller|Display" | grep -iE "amd|radeon" &> /dev/null; then
|
||||
has_amd_gpu='true'
|
||||
echo -e "${GREEN}✓${RESET} AMD GPU detected — ROCm acceleration will be configured automatically when AI Assistant is installed.\\n"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
# Write detected GPU type to a marker file the admin container can read. The admin
|
||||
# container lacks lspci and AMD GPUs don't register a Docker runtime, so this is the
|
||||
# only reliable way for the admin to know an AMD GPU is present at install time.
|
||||
local gpu_marker_path="${NOMAD_DIR}/storage/.nomad-gpu-type"
|
||||
if command -v nvidia-smi &> /dev/null; then
|
||||
echo 'nvidia' | sudo tee "${gpu_marker_path}" > /dev/null 2>&1 || true
|
||||
elif [[ "${has_amd_gpu}" == 'true' ]]; then
|
||||
echo 'amd' | sudo tee "${gpu_marker_path}" > /dev/null 2>&1 || true
|
||||
else
|
||||
sudo rm -f "${gpu_marker_path}" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo -e "${YELLOW}===========================================${RESET}\\n"
|
||||
|
||||
|
||||
# Summary
|
||||
if command -v nvidia-smi &> /dev/null && docker info 2>/dev/null | grep -q "nvidia"; then
|
||||
echo -e "${GREEN}#${RESET} GPU acceleration is properly configured! The AI Assistant will use your GPU.\\n"
|
||||
elif [[ "${has_amd_gpu}" == 'true' ]]; then
|
||||
echo -e "${GREEN}#${RESET} GPU acceleration will be enabled (AMD/ROCm) when AI Assistant is installed from the dashboard.\\n"
|
||||
else
|
||||
echo -e "${YELLOW}#${RESET} GPU acceleration not detected. The AI Assistant will run in CPU-only mode.\\n"
|
||||
if command -v nvidia-smi &> /dev/null && ! docker info 2>/dev/null | grep -q "nvidia"; then
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "project-nomad",
|
||||
"version": "1.31.0",
|
||||
"version": "1.32.0-rc.1",
|
||||
"description": "\"",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
|
|
|
|||
Loading…
Reference in New Issue