diff --git a/Dockerfile b/Dockerfile index 8850e23..0dcc184 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,31 @@ FROM base ARG VERSION=dev ARG BUILD_DATE ARG VCS_REF +ARG TARGETARCH + +# go-pmtiles (regional map extracts). Pinned so the CLI's stdout format stays +# in sync with parseDryRunOutput(). +ARG PMTILES_VERSION=1.30.2 +# Upstream releases don't ship a checksums file, so pin per-arch SHA256 here. +# When bumping PMTILES_VERSION, regenerate these with: +# curl -fsSL | sha256sum +ARG PMTILES_SHA256_AMD64=2cd3aa18868297fc88425038f794efdc0995e0275f4ca16fa496dd79e245a40c +ARG PMTILES_SHA256_ARM64=804cdf071834e1156af554c1a26cc42b56b9cde5a2db9c6e3653d16fb846d5fa +RUN set -eux; \ + case "${TARGETARCH:-amd64}" in \ + amd64) PMTILES_ARCH=x86_64; PMTILES_SHA256="${PMTILES_SHA256_AMD64}" ;; \ + arm64) PMTILES_ARCH=arm64; PMTILES_SHA256="${PMTILES_SHA256_ARM64}" ;; \ + *) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac; \ + TARBALL="go-pmtiles_${PMTILES_VERSION}_Linux_${PMTILES_ARCH}.tar.gz"; \ + cd /tmp; \ + curl -fsSL -o "$TARBALL" \ + "https://github.com/protomaps/go-pmtiles/releases/download/v${PMTILES_VERSION}/${TARBALL}"; \ + echo "${PMTILES_SHA256} ${TARBALL}" | sha256sum -c -; \ + tar -xzf "$TARBALL" -C /usr/local/bin pmtiles; \ + rm -f "$TARBALL"; \ + chmod +x /usr/local/bin/pmtiles; \ + /usr/local/bin/pmtiles version # Labels LABEL org.opencontainers.image.title="Project N.O.M.A.D" \ diff --git a/admin/adonisrc.ts b/admin/adonisrc.ts index a091ce2..34586ca 100644 --- a/admin/adonisrc.ts +++ b/admin/adonisrc.ts @@ -107,6 +107,10 @@ export default defineConfig({ pattern: 'resources/views/**/*.edge', reloadServer: false, }, + { + pattern: 'resources/geodata/**/*.geojson', + reloadServer: false, + }, { pattern: 'public/**', reloadServer: false, diff --git a/admin/app/controllers/maps_controller.ts b/admin/app/controllers/maps_controller.ts index dd93a8b..097d9fb 100644 --- a/admin/app/controllers/maps_controller.ts +++ b/admin/app/controllers/maps_controller.ts @@ -4,6 +4,8 @@ import { assertNotPrivateUrl, downloadCollectionValidator, filenameParamValidator, + mapExtractPreflightValidator, + mapExtractValidator, remoteDownloadValidator, remoteDownloadValidatorOptional, } from '#validators/common' @@ -87,6 +89,28 @@ export default class MapsController { } } + async listCountries({}: HttpContext) { + return { countries: await this.mapService.listCountries() } + } + + async listCountryGroups({}: HttpContext) { + return { groups: await this.mapService.listCountryGroups() } + } + + async extractPreflight({ request }: HttpContext) { + const payload = await request.validateUsing(mapExtractPreflightValidator) + return await this.mapService.extractPreflight(payload) + } + + async extractRegion({ request }: HttpContext) { + const payload = await request.validateUsing(mapExtractValidator) + const result = await this.mapService.extractRegion(payload) + return { + message: 'Extract started successfully', + ...result, + } + } + async styles({ request, response }: HttpContext) { // Automatically ensure base assets are present before generating styles const baseAssetsExist = await this.mapService.ensureBaseAssets() diff --git a/admin/app/jobs/run_extract_pmtiles_job.ts b/admin/app/jobs/run_extract_pmtiles_job.ts new file mode 100644 index 0000000..73c4eed --- /dev/null +++ b/admin/app/jobs/run_extract_pmtiles_job.ts @@ -0,0 +1,294 @@ +import { Job, UnrecoverableError } from 'bullmq' +import { spawn, ChildProcess } from 'child_process' +import { createHash } from 'crypto' +import { readdir, stat } from 'fs/promises' +import { basename, dirname, join } from 'path' +import { QueueService } from '#services/queue_service' +import logger from '@adonisjs/core/services/logger' +import { DownloadProgressData } from '../../types/downloads.js' +import { PMTILES_BINARY_PATH, buildPmtilesExtractArgs } from '../../constants/map_regions.js' +import { deleteFileIfExists } from '../utils/fs.js' + +export interface RunExtractPmtilesJobParams { + sourceUrl: string + outputFilepath: string + /** Path to a GeoJSON FeatureCollection file passed to `pmtiles extract --region`. */ + regionFilepath: string + maxzoom?: number + /** Hint for progress reporting; obtained from `pmtiles extract --dry-run` preflight */ + estimatedBytes?: number + filetype: 'map' + title?: string + resourceMetadata?: { + resource_id: string + version: string + collection_ref: string | null + } +} + +export class RunExtractPmtilesJob { + static get queue() { + return 'pmtiles-extract' + } + + static get key() { + return 'run-pmtiles-extract' + } + + /** In-memory registry of active child processes so in-process cancels can SIGTERM them */ + static childProcesses: Map = new Map() + + static getJobId(sourceUrl: string, regionFilepath: string, maxzoom?: number): string { + const payload = JSON.stringify({ sourceUrl, regionFilepath, maxzoom: maxzoom ?? null }) + return createHash('sha256').update(payload).digest('hex').slice(0, 16) + } + + /** Redis key used to signal cancellation across processes */ + static cancelKey(jobId: string): string { + return `nomad:download:pmtiles-cancel:${jobId}` + } + + static async signalCancel(jobId: string): Promise { + const queueService = new QueueService() + const queue = queueService.getQueue(this.queue) + const client = await queue.client + await client.set(this.cancelKey(jobId), '1', 'EX', 300) + } + + /** Awaits job.updateProgress and swallows BullMQ stale-job errors (code -1), + * which occur when the job was removed from Redis (e.g. cancelled) between + * the await being issued and the Redis write completing. Anything else + * re-throws so it's caught by the surrounding try rather than becoming an + * unhandled rejection. */ + private async safeUpdateProgress(job: Job, progress: DownloadProgressData): Promise { + try { + await job.updateProgress(progress) + } catch (err: any) { + if (err?.code !== -1) throw err + } + } + + async handle(job: Job) { + const params = job.data as RunExtractPmtilesJobParams + const { sourceUrl, outputFilepath, regionFilepath, maxzoom, estimatedBytes } = params + + logger.info( + `[RunExtractPmtilesJob] Starting extract: source=${sourceUrl} region=${regionFilepath} ` + + `maxzoom=${maxzoom ?? 'source-max'} out=${outputFilepath}` + ) + + const queueService = new QueueService() + const cancelRedis = await queueService.getQueue(RunExtractPmtilesJob.queue).client + + let userCancelled = false + let proc: ChildProcess | null = null + let lastReportedBytes = -1 + + // One 2s tick polls the Redis cancel signal and reads file-size for progress. pmtiles + // writes incrementally but rewrites directories near the end so progress isn't strictly + // monotonic — we cap at 99% and skip emit when bytes are unchanged to avoid Redis chatter. + const tick = setInterval(async () => { + try { + const val = await cancelRedis.get(RunExtractPmtilesJob.cancelKey(job.id!)) + if (val) { + await cancelRedis.del(RunExtractPmtilesJob.cancelKey(job.id!)) + userCancelled = true + proc?.kill('SIGTERM') + } + } catch { + // Redis errors non-fatal — in-memory handle also covers same-process cancels + } + + try { + const fileStat = await stat(outputFilepath) + const downloadedBytes = Number(fileStat.size) + if (downloadedBytes === lastReportedBytes) return + lastReportedBytes = downloadedBytes + + const totalBytes = estimatedBytes ?? 0 + const percent = + totalBytes > 0 ? Math.min(99, Math.floor((downloadedBytes / totalBytes) * 100)) : 0 + + await this.safeUpdateProgress(job, { + percent, + downloadedBytes, + totalBytes, + lastProgressTime: Date.now(), + } as DownloadProgressData) + } catch { + // File doesn't exist yet (subprocess still setting up) + } + }, 2000) + + try { + const args = buildPmtilesExtractArgs({ + sourceUrl, + outputFilepath, + regionFilepath, + maxzoom, + downloadThreads: 8, + overfetch: 0.2, + }) + proc = spawn(PMTILES_BINARY_PATH, args, { stdio: ['ignore', 'pipe', 'pipe'] }) + RunExtractPmtilesJob.childProcesses.set(job.id!, proc) + + proc.stdout?.on('data', (chunk) => { + logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`) + }) + proc.stderr?.on('data', (chunk) => { + logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`) + }) + + const exitCode: number = await new Promise((resolve, reject) => { + proc!.on('close', (code) => resolve(code ?? -1)) + proc!.on('error', (err) => reject(err)) + }) + + if (exitCode !== 0) { + await deleteFileIfExists(outputFilepath) + if (userCancelled) { + throw new UnrecoverableError(`Extract cancelled by user (exit ${exitCode})`) + } + throw new Error(`pmtiles extract exited with code ${exitCode}`) + } + + // Final progress bump — tick caps at 99 so the UI doesn't flicker to 100 mid-extract + const finalStat = await stat(outputFilepath) + await this.safeUpdateProgress(job, { + percent: 100, + downloadedBytes: Number(finalStat.size), + totalBytes: estimatedBytes ?? Number(finalStat.size), + lastProgressTime: Date.now(), + } as DownloadProgressData) + + // Reuse the HTTP download path's post-download hook so the file is registered and + // the previous version (if any) is deleted + await this.onComplete(params) + + logger.info( + `[RunExtractPmtilesJob] Completed extract: out=${outputFilepath} size=${finalStat.size} bytes` + ) + + return { sourceUrl, outputFilepath } + } catch (error: any) { + if (userCancelled && !(error instanceof UnrecoverableError)) { + throw new UnrecoverableError(`Extract cancelled: ${error.message ?? error}`) + } + throw error + } finally { + clearInterval(tick) + RunExtractPmtilesJob.childProcesses.delete(job.id!) + } + } + + private async onComplete(params: RunExtractPmtilesJobParams) { + if (!params.resourceMetadata) return + + const [{ default: InstalledResource }, { DateTime }, fsUtils] = await Promise.all([ + import('#models/installed_resource'), + import('luxon'), + import('../utils/fs.js'), + ]) + + const fileStat = await fsUtils.getFileStatsIfExists(params.outputFilepath) + + const existing = await InstalledResource.query() + .where('resource_id', params.resourceMetadata.resource_id) + .where('resource_type', 'map') + .first() + const oldFilePath = existing?.file_path ?? null + + await InstalledResource.updateOrCreate( + { + resource_id: params.resourceMetadata.resource_id, + resource_type: 'map', + }, + { + version: params.resourceMetadata.version, + collection_ref: params.resourceMetadata.collection_ref, + url: params.sourceUrl, + file_path: params.outputFilepath, + file_size_bytes: fileStat ? Number(fileStat.size) : null, + installed_at: DateTime.now(), + } + ) + + if (oldFilePath && oldFilePath !== params.outputFilepath) { + try { + await fsUtils.deleteFileIfExists(oldFilePath) + } catch (err) { + logger.warn(`[RunExtractPmtilesJob] Failed to delete old file ${oldFilePath}: ${err}`) + } + } + + // Fallback: scan the pmtiles dir for orphans with the same resource_id that the DB + // lookup above didn't catch — e.g. a prior extract crashed before writing its + // InstalledResource row, or an earlier bug wrote a file without registering it. + // Matches both curated (`_YYYY-MM.pmtiles`) and regional (`_YYYYMMDD_zN.pmtiles`) + // naming — prefix-only so new filename formats don't silently miss. + const dir = dirname(params.outputFilepath) + const keepName = basename(params.outputFilepath) + const prefix = `${params.resourceMetadata.resource_id}_` + try { + const entries = await readdir(dir) + for (const entry of entries) { + if (entry === keepName || !entry.endsWith('.pmtiles')) continue + if (!entry.startsWith(prefix)) continue + const orphanPath = join(dir, entry) + if (orphanPath === oldFilePath) continue + try { + await fsUtils.deleteFileIfExists(orphanPath) + logger.info(`[RunExtractPmtilesJob] Pruned orphan pmtiles ${orphanPath}`) + } catch (err) { + logger.warn(`[RunExtractPmtilesJob] Failed to prune orphan ${orphanPath}: ${err}`) + } + } + } catch (err) { + logger.warn(`[RunExtractPmtilesJob] Directory scan for orphans failed: ${err}`) + } + } + + static async getById(jobId: string): Promise { + const queueService = new QueueService() + const queue = queueService.getQueue(this.queue) + return await queue.getJob(jobId) + } + + static async dispatch(params: RunExtractPmtilesJobParams) { + const queueService = new QueueService() + const queue = queueService.getQueue(this.queue) + const jobId = this.getJobId(params.sourceUrl, params.regionFilepath, params.maxzoom) + + const existing = await queue.getJob(jobId) + if (existing) { + const state = await existing.getState() + if (state === 'active' || state === 'waiting' || state === 'delayed') { + return { + job: existing, + created: false, + message: `Extract job already exists for these params`, + } + } + // Stale (completed/failed) — remove so we can re-dispatch under the same deterministic id + try { + await existing.remove() + } catch { + // Already gone or locked — add() below will still report a meaningful error + } + } + + // Fewer attempts than HTTP downloads — a failed extract usually means the source URL + // rotated or the CDN is throttling, and resuming mid-extract isn't supported by the CLI + const job = await queue.add(this.key, params, { + jobId, + attempts: 3, + backoff: { type: 'exponential', delay: 60000 }, + removeOnComplete: true, + }) + return { + job, + created: true, + message: `Dispatched pmtiles extract job`, + } + } +} diff --git a/admin/app/services/countries_service.ts b/admin/app/services/countries_service.ts new file mode 100644 index 0000000..b6ebf0f --- /dev/null +++ b/admin/app/services/countries_service.ts @@ -0,0 +1,308 @@ +import { access, readFile, writeFile, mkdir } from 'fs/promises' +import { join, resolve } from 'path' +import { createHash } from 'crypto' +import { tmpdir } from 'os' +import logger from '@adonisjs/core/services/logger' +import type { Country, CountryCode, CountryGroup } from '../../types/maps.js' + +interface NEFeature { + type: 'Feature' + properties: Record + geometry: unknown +} + +interface NEFeatureCollection { + type: 'FeatureCollection' + features: NEFeature[] +} + +const COUNTRY_GEOJSON_PATH = join( + process.cwd(), + 'resources', + 'geodata', + 'ne_50m_admin_0_countries.geojson' +) + +// Natural Earth country polygons are land-only (no territorial waters), so a +// strict intersect leaves tiles fully over the ocean out of the extract — +// coastal cities render as grey off their coast. Inflate each polygon outward +// by ~11 km to pull in adjacent tiles without ballooning the extract size. +const REGION_BUFFER_DEGREES = 0.1 + +const GROUP_ORDER = [ + 'north-america', + 'south-america', + 'europe', + 'africa', + 'asia', + 'oceania', +] + +const GROUP_META: Record = { + 'North America': { + id: 'north-america', + name: 'North America', + description: 'All countries in North America and the Caribbean.', + }, + 'South America': { + id: 'south-america', + name: 'South America', + description: 'All countries in South America.', + }, + Europe: { + id: 'europe', + name: 'Europe', + description: 'All countries in Europe.', + }, + Africa: { + id: 'africa', + name: 'Africa', + description: 'All countries in Africa.', + }, + Asia: { + id: 'asia', + name: 'Asia', + description: 'All countries in Asia.', + }, + Oceania: { + id: 'oceania', + name: 'Oceania', + description: 'Australia, New Zealand, and Pacific island nations.', + }, +} + +export class CountriesService { + private static instance: CountriesService | null = null + private loadPromise: Promise | null = null + private countries: Country[] = [] + private byCode: Map = new Map() + private groups: CountryGroup[] = [] + + static getInstance(): CountriesService { + if (!this.instance) { + this.instance = new CountriesService() + } + return this.instance + } + + private async ensureLoaded(): Promise { + if (this.byCode.size > 0) return + if (!this.loadPromise) { + this.loadPromise = this.load() + } + await this.loadPromise + } + + private async load(): Promise { + const raw = await readFile(COUNTRY_GEOJSON_PATH, 'utf8') + const fc = JSON.parse(raw) as NEFeatureCollection + + // Natural Earth reuses a sovereign state's ISO_A2 for its dependencies + // (e.g. AU covers both Australia and Australian territories). Sort so the + // sovereign mainland wins the ISO-code slot, and skip any subsequent + // same-code dependency — otherwise the "AU" entry ends up being some tiny + // island territory. + const sortedFeatures = [...fc.features].sort((a, b) => typeRank(a) - typeRank(b)) + + const countries: Country[] = [] + const byCode = new Map() + const groupCodes: Record = {} + + for (const feature of sortedFeatures) { + const p = feature.properties + const code = resolveIso2(p) + if (!code) continue + if (byCode.has(code)) continue + + const continent = typeof p.CONTINENT === 'string' ? p.CONTINENT : 'Other' + if (continent === 'Antarctica' || continent === 'Seven seas (open ocean)') continue + + const country: Country = { + code, + code3: resolveIso3(p) ?? code, + name: typeof p.NAME === 'string' ? p.NAME : code, + continent, + subregion: typeof p.SUBREGION === 'string' ? p.SUBREGION : continent, + population: typeof p.POP_EST === 'number' ? p.POP_EST : 0, + } + + countries.push(country) + byCode.set(code, { country, feature }) + + if (GROUP_META[continent]) { + const groupId = GROUP_META[continent].id + if (!groupCodes[groupId]) groupCodes[groupId] = [] + groupCodes[groupId].push(code) + } + } + + countries.sort((a, b) => a.name.localeCompare(b.name)) + + const groups: CountryGroup[] = GROUP_ORDER.flatMap((groupId) => { + const meta = Object.values(GROUP_META).find((m) => m.id === groupId) + if (!meta) return [] + const codes = (groupCodes[groupId] ?? []).slice().sort() + if (codes.length === 0) return [] + return [{ id: meta.id, name: meta.name, description: meta.description, countries: codes }] + }) + + this.countries = countries + this.byCode = byCode + this.groups = groups + + logger.info( + `[CountriesService] Loaded ${countries.length} countries across ${groups.length} groups` + ) + } + + async list(): Promise { + await this.ensureLoaded() + return this.countries + } + + async listGroups(): Promise { + await this.ensureLoaded() + return this.groups + } + + /** Throws when a supplied code does not map to a known country. */ + async resolveCodes(codes: CountryCode[]): Promise { + await this.ensureLoaded() + const normalized = [...new Set(codes.map((c) => c.toUpperCase()))].sort() + const unknown = normalized.filter((c) => !this.byCode.has(c)) + if (unknown.length > 0) { + throw new Error(`Unknown country code(s): ${unknown.join(', ')}`) + } + return normalized + } + + /** + * Filename is keyed on a hash of the sorted ISO codes + buffer size so + * repeated calls with the same selection reuse the same path, and bumping + * the buffer auto-invalidates stale files. + */ + async writeRegionFile(codes: CountryCode[]): Promise { + await this.ensureLoaded() + const resolved = await this.resolveCodes(codes) + const key = `b${REGION_BUFFER_DEGREES}:${resolved.join(',')}` + const hash = createHash('sha1').update(key).digest('hex').slice(0, 12) + + const dir = resolve(tmpdir(), 'nomad-pmtiles-regions') + await mkdir(dir, { recursive: true }) + const filepath = join(dir, `region-${hash}.geojson`) + + try { + await access(filepath) + return filepath + } catch {} + + const fc = { + type: 'FeatureCollection', + features: resolved.map((code) => { + const entry = this.byCode.get(code)! + return { + type: 'Feature', + properties: { iso: code, name: entry.country.name }, + geometry: bufferGeometry(entry.feature.geometry, REGION_BUFFER_DEGREES), + } + }), + } + + await writeFile(filepath, JSON.stringify(fc)) + return filepath + } +} + +function typeRank(f: NEFeature): number { + const t = typeof f.properties.TYPE === 'string' ? f.properties.TYPE : '' + if (t === 'Sovereign country') return 0 + if (t === 'Country') return 1 + if (t === 'Sovereignty') return 2 + if (t === 'Disputed') return 3 + if (t === 'Dependency') return 4 + return 5 +} + +function resolveIso2(p: Record): CountryCode | null { + // Natural Earth's ISO_A2 sometimes holds political escapes like "CN-TW" for + // Taiwan or "-99" for countries involved in disputes. Only accept clean + // 2-letter codes; fall back to ISO_A2_EH (which reliably has the real code). + const primary = typeof p.ISO_A2 === 'string' ? p.ISO_A2 : null + if (primary && /^[A-Z]{2}$/i.test(primary)) return primary.toUpperCase() + const fallback = typeof p.ISO_A2_EH === 'string' ? p.ISO_A2_EH : null + if (fallback && /^[A-Z]{2}$/i.test(fallback)) return fallback.toUpperCase() + return null +} + +/** + * Inflate each polygon ring outward by `buffer` degrees via per-vertex + * averaged-normal offset. Not geodesically accurate — but at small buffers + * (<= 0.2°) it's within a few percent of a proper geodesic buffer at + * country scale, which is plenty for tile-inclusion purposes. + */ +function bufferGeometry(geometry: unknown, buffer: number): unknown { + const geom = geometry as { type: string; coordinates: any } + if (geom?.type === 'Polygon') { + return { type: 'Polygon', coordinates: bufferPolygonRings(geom.coordinates, buffer) } + } + if (geom?.type === 'MultiPolygon') { + return { + type: 'MultiPolygon', + coordinates: geom.coordinates.map((poly: number[][][]) => + bufferPolygonRings(poly, buffer) + ), + } + } + return geometry +} + +function bufferPolygonRings(rings: number[][][], buffer: number): number[][][] { + return rings.map((ring) => bufferRing(ring, buffer)) +} + +function bufferRing(ring: number[][], buffer: number): number[][] { + if (ring.length < 4) return ring + const sign = signedArea(ring) > 0 ? 1 : -1 + const n = ring.length - 1 + const out: number[][] = [] + for (let i = 0; i < n; i++) { + const prev = ring[(i - 1 + n) % n] + const curr = ring[i] + const next = ring[(i + 1) % n] + const e1x = curr[0] - prev[0] + const e1y = curr[1] - prev[1] + const e2x = next[0] - curr[0] + const e2y = next[1] - curr[1] + const l1 = Math.hypot(e1x, e1y) || 1 + const l2 = Math.hypot(e2x, e2y) || 1 + const n1x = (e1y / l1) * sign + const n1y = (-e1x / l1) * sign + const n2x = (e2y / l2) * sign + const n2y = (-e2x / l2) * sign + const sumX = n1x + n2x + const sumY = n1y + n2y + const sl = Math.hypot(sumX, sumY) || 1 + out.push([curr[0] + (sumX / sl) * buffer, curr[1] + (sumY / sl) * buffer]) + } + out.push(out[0]) + return out +} + +function signedArea(ring: number[][]): number { + let a = 0 + for (let i = 0; i < ring.length - 1; i++) { + a += ring[i][0] * ring[i + 1][1] - ring[i + 1][0] * ring[i][1] + } + return a / 2 +} + +function resolveIso3(p: Record): string | null { + const primary = typeof p.ISO_A3 === 'string' ? p.ISO_A3 : null + if (primary && primary !== '-99') return primary.toUpperCase() + const fallback = typeof p.ISO_A3_EH === 'string' ? p.ISO_A3_EH : null + if (fallback && fallback !== '-99') return fallback.toUpperCase() + const adm = typeof p.ADM0_A3 === 'string' ? p.ADM0_A3 : null + if (adm && adm !== '-99') return adm.toUpperCase() + return null +} + diff --git a/admin/app/services/download_service.ts b/admin/app/services/download_service.ts index bd9076c..91b8288 100644 --- a/admin/app/services/download_service.ts +++ b/admin/app/services/download_service.ts @@ -1,13 +1,19 @@ import { inject } from '@adonisjs/core' import { QueueService } from './queue_service.js' import { RunDownloadJob } from '#jobs/run_download_job' +import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job' +import type { RunExtractPmtilesJobParams } from '#jobs/run_extract_pmtiles_job' import { DownloadModelJob } from '#jobs/download_model_job' import { DownloadJobWithProgress, DownloadProgressData } from '../../types/downloads.js' +import type { Job, Queue } from 'bullmq' import { normalize } from 'path' import { deleteFileIfExists } from '../utils/fs.js' import transmit from '@adonisjs/transmit/services/main' import { BROADCAST_CHANNELS } from '../../constants/broadcast.js' +type FileJobState = 'waiting' | 'active' | 'delayed' | 'failed' +type TaggedJob = { job: Job; state: FileJobState } + @inject() export class DownloadService { constructor(private queueService: QueueService) {} @@ -26,27 +32,32 @@ export class DownloadService { return { percent: parseInt(String(progress), 10) || 0 } } - async listDownloadJobs(filetype?: string): Promise { - // Get regular file download jobs (zim, map, etc.) — query each state separately so we can - // tag each job with its actual BullMQ state rather than guessing from progress data. - const queue = this.queueService.getQueue(RunDownloadJob.queue) - type FileJobState = 'waiting' | 'active' | 'delayed' | 'failed' - - const [waitingJobs, activeJobs, delayedJobs, failedJobs] = await Promise.all([ + /** Fetch all non-completed jobs from a queue, tagged with their current BullMQ state */ + private async fetchJobsWithStates(queueName: string): Promise { + const queue = this.queueService.getQueue(queueName) + const [waiting, active, delayed, failed] = await Promise.all([ queue.getJobs(['waiting']), queue.getJobs(['active']), queue.getJobs(['delayed']), queue.getJobs(['failed']), ]) - - const taggedFileJobs: Array<{ job: (typeof waitingJobs)[0]; state: FileJobState }> = [ - ...waitingJobs.map((j) => ({ job: j, state: 'waiting' as const })), - ...activeJobs.map((j) => ({ job: j, state: 'active' as const })), - ...delayedJobs.map((j) => ({ job: j, state: 'delayed' as const })), - ...failedJobs.map((j) => ({ job: j, state: 'failed' as const })), + return [ + ...waiting.map((j) => ({ job: j, state: 'waiting' as const })), + ...active.map((j) => ({ job: j, state: 'active' as const })), + ...delayed.map((j) => ({ job: j, state: 'delayed' as const })), + ...failed.map((j) => ({ job: j, state: 'failed' as const })), ] + } - const fileDownloads = taggedFileJobs.map(({ job, state }) => { + async listDownloadJobs(filetype?: string): Promise { + const modelQueue = this.queueService.getQueue(DownloadModelJob.queue) + const [fileTagged, extractTagged, modelJobs] = await Promise.all([ + this.fetchJobsWithStates(RunDownloadJob.queue), + this.fetchJobsWithStates(RunExtractPmtilesJob.queue), + modelQueue.getJobs(['waiting', 'active', 'delayed', 'failed']), + ]) + + const fileDownloads = fileTagged.map(({ job, state }) => { const parsed = this.parseProgress(job.progress) return { jobId: job.id!.toString(), @@ -63,26 +74,36 @@ export class DownloadService { } }) - // Get Ollama model download jobs - const modelQueue = this.queueService.getQueue(DownloadModelJob.queue) - const modelJobs = await modelQueue.getJobs(['waiting', 'active', 'delayed', 'failed']) + const extractDownloads = extractTagged.map(({ job, state }) => { + const parsed = this.parseProgress(job.progress) + return { + jobId: job.id!.toString(), + url: job.data.sourceUrl, + progress: parsed.percent, + filepath: normalize(job.data.outputFilepath), + filetype: job.data.filetype || 'map', + title: job.data.title || undefined, + downloadedBytes: parsed.downloadedBytes, + totalBytes: parsed.totalBytes || job.data.estimatedBytes || undefined, + lastProgressTime: parsed.lastProgressTime, + status: state, + failedReason: job.failedReason || undefined, + } + }) const modelDownloads = modelJobs.map((job) => ({ jobId: job.id!.toString(), - url: job.data.modelName || 'Unknown Model', // Use model name as url + url: job.data.modelName || 'Unknown Model', progress: parseInt(job.progress.toString(), 10), - filepath: job.data.modelName || 'Unknown Model', // Use model name as filepath + filepath: job.data.modelName || 'Unknown Model', filetype: 'model', status: (job.failedReason ? 'failed' : 'active') as 'active' | 'failed', failedReason: job.failedReason || undefined, })) - const allDownloads = [...fileDownloads, ...modelDownloads] - - // Filter by filetype if specified + const allDownloads = [...fileDownloads, ...extractDownloads, ...modelDownloads] const filtered = allDownloads.filter((job) => !filetype || job.filetype === filetype) - // Sort: active downloads first (by progress desc), then failed at the bottom return filtered.sort((a, b) => { if (a.status === 'failed' && b.status !== 'failed') return 1 if (a.status !== 'failed' && b.status === 'failed') return -1 @@ -91,7 +112,11 @@ export class DownloadService { } async removeFailedJob(jobId: string): Promise { - for (const queueName of [RunDownloadJob.queue, DownloadModelJob.queue]) { + for (const queueName of [ + RunDownloadJob.queue, + RunExtractPmtilesJob.queue, + DownloadModelJob.queue, + ]) { const queue = this.queueService.getQueue(queueName) const job = await queue.getJob(jobId) if (job) { @@ -113,7 +138,6 @@ export class DownloadService { } async cancelJob(jobId: string): Promise<{ success: boolean; message: string }> { - // Try the file download queue first (the original PR #554 path) const queue = this.queueService.getQueue(RunDownloadJob.queue) const job = await queue.getJob(jobId) @@ -121,7 +145,13 @@ export class DownloadService { return await this._cancelFileDownloadJob(jobId, job, queue) } - // Fall through to the model download queue + const extractQueue = this.queueService.getQueue(RunExtractPmtilesJob.queue) + const extractJob = await extractQueue.getJob(jobId) + + if (extractJob) { + return await this._cancelExtractJob(jobId, extractJob, extractQueue) + } + const modelQueue = this.queueService.getQueue(DownloadModelJob.queue) const modelJob = await modelQueue.getJob(jobId) @@ -129,11 +159,37 @@ export class DownloadService { return await this._cancelModelDownloadJob(jobId, modelJob, modelQueue) } - // Not found in either queue return { success: true, message: 'Job not found (may have already completed)' } } - /** Cancel a content download (zim, map, pmtiles, etc.) — original PR #554 logic */ + private async _cancelExtractJob( + jobId: string, + job: Job, + queue: Queue + ): Promise<{ success: boolean; message: string }> { + const outputFilepath = job.data.outputFilepath + + await RunExtractPmtilesJob.signalCancel(jobId) + + // Same-process fallback when worker and API share a process + RunExtractPmtilesJob.childProcesses.get(jobId)?.kill('SIGTERM') + RunExtractPmtilesJob.childProcesses.delete(jobId) + + await this._pollForTerminalState(job, jobId) + await this._removeJobWithLockFallback(job, queue, RunExtractPmtilesJob.queue, jobId) + + if (outputFilepath) { + try { + await deleteFileIfExists(outputFilepath) + } catch { + // File may not exist yet (subprocess may not have opened it) + } + } + + return { success: true, message: 'Extract cancelled and partial file deleted' } + } + + /** Cancel a content download (zim, map, pmtiles, etc.) */ private async _cancelFileDownloadJob( jobId: string, job: any, diff --git a/admin/app/services/map_service.ts b/admin/app/services/map_service.ts index c9902a3..c1157b0 100644 --- a/admin/app/services/map_service.ts +++ b/admin/app/services/map_service.ts @@ -16,11 +16,35 @@ import { import { join, resolve, sep } from 'path' import urlJoin from 'url-join' import { RunDownloadJob } from '#jobs/run_download_job' +import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job' import logger from '@adonisjs/core/services/logger' import { assertNotPrivateUrl } from '#validators/common' import InstalledResource from '#models/installed_resource' import { CollectionManifestService } from './collection_manifest_service.js' import type { CollectionWithStatus, MapsSpec } from '../../types/collections.js' +import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps.js' +import { + EXTRACT_DEFAULT_MAX_ZOOM, + EXTRACT_MAX_ZOOM, + EXTRACT_MIN_ZOOM, + PMTILES_BINARY_PATH, + WORLD_BASEMAP_FILENAME, + WORLD_BASEMAP_MAX_ZOOM, + WORLD_BASEMAP_SOURCE_NAME, + buildPmtilesExtractArgs, +} from '../../constants/map_regions.js' +import { CountriesService } from './countries_service.js' +import { execFile } from 'child_process' +import { createHash, randomBytes } from 'crypto' +import { tmpdir } from 'os' +import { promisify } from 'util' + +const execFileAsync = promisify(execFile) +const DRY_RUN_TIMEOUT_MS = 60_000 +const DRY_RUN_MAX_BUFFER = 256 * 1024 +// Real extract of z0-5 world tiles; generous to tolerate slow/metered links +// since a failure leaves the map grey for uncovered regions. +const WORLD_BASEMAP_EXTRACT_TIMEOUT_MS = 5 * 60_000 const PROTOMAPS_BUILDS_METADATA_URL = 'https://build-metadata.protomaps.dev/builds.json' const PROTOMAPS_BUILD_BASE_URL = 'https://build.protomaps.com' @@ -53,10 +77,15 @@ export class MapService implements IMapService { private readonly baseAssetsTarFile = 'base-assets.tar.gz' private readonly baseDirPath = join(process.cwd(), this.mapStoragePath) private baseAssetsExistCache: boolean | null = null + private worldBasemapReady = false + private worldBasemapInFlight: Promise | null = null async listRegions() { const files = (await this.listAllMapStorageItems()).filter( - (item) => item.type === 'file' && item.name.endsWith('.pmtiles') + (item) => + item.type === 'file' && + item.name.endsWith('.pmtiles') && + item.name !== WORLD_BASEMAP_FILENAME ) return { @@ -327,11 +356,76 @@ export class MapService implements IMapService { async ensureBaseAssets(): Promise { const exists = await this.checkBaseAssetsExist() - if (exists) { - return true + if (!exists) { + const downloaded = await this.downloadBaseAssets() + if (!downloaded) return false } - return await this.downloadBaseAssets() + try { + await this.ensureWorldBasemap() + } catch (err) { + logger.warn(`[MapService] World basemap setup failed, continuing without it: ${err}`) + } + + return true + } + + /** + * Extract a low-zoom global basemap once so the map isn't grey outside a + * regional extract's polygon. Cheap (~15 MB, a handful of HTTP range + * requests) and layered underneath regional sources at render time. + * + * Memoizes success in-process, and de-duplicates concurrent callers via a + * shared in-flight promise so two simultaneous `/maps` requests on a cold + * start don't both launch `pmtiles extract` against the same output path. + */ + private async ensureWorldBasemap(): Promise { + if (this.worldBasemapReady) return + if (this.worldBasemapInFlight) return this.worldBasemapInFlight + this.worldBasemapInFlight = this._setupWorldBasemap().finally(() => { + this.worldBasemapInFlight = null + }) + return this.worldBasemapInFlight + } + + private async _setupWorldBasemap(): Promise { + const basePath = resolve(join(this.baseDirPath, 'pmtiles')) + const filepath = resolve(join(basePath, WORLD_BASEMAP_FILENAME)) + if (!filepath.startsWith(basePath + sep)) { + throw new Error('Invalid world basemap path') + } + + await ensureDirectoryExists(basePath) + + const existing = await getFileStatsIfExists(filepath) + if (existing && Number(existing.size) > 0) { + this.worldBasemapReady = true + return + } + + const info = await this.getGlobalMapInfo() + const args = buildPmtilesExtractArgs({ + sourceUrl: info.url, + outputFilepath: filepath, + maxzoom: WORLD_BASEMAP_MAX_ZOOM, + downloadThreads: 4, + }) + + logger.info( + `[MapService] Extracting world basemap (z0-${WORLD_BASEMAP_MAX_ZOOM}) from ${info.url}` + ) + try { + await execFileAsync(PMTILES_BINARY_PATH, args, { + timeout: WORLD_BASEMAP_EXTRACT_TIMEOUT_MS, + maxBuffer: DRY_RUN_MAX_BUFFER, + }) + this.worldBasemapReady = true + } catch (err: any) { + await deleteFileIfExists(filepath) + throw new Error( + `pmtiles extract for world basemap failed: ${err.message}. stderr: ${err.stderr ?? ''}` + ) + } } private async checkBaseAssetsExist(useCache: boolean = true): Promise { @@ -367,6 +461,19 @@ export class MapService implements IMapService { const sources: BaseStylesFile['sources'][] = [] const baseUrl = this.getPublicFileBaseUrl(host, 'pmtiles', protocol) + // World basemap goes first so its layers render underneath regional extracts. + // Only emitted when ensureWorldBasemap() succeeded — otherwise the style would + // reference a file that doesn't exist and produce 404s on every tile request. + if (this.worldBasemapReady) { + const worldSource: BaseStylesFile['sources'] = {} + worldSource[WORLD_BASEMAP_SOURCE_NAME] = { + type: 'vector', + attribution: PMTILES_ATTRIBUTION, + url: `pmtiles://${urlJoin(baseUrl, WORLD_BASEMAP_FILENAME)}`, + } + sources.push(worldSource) + } + for (const region of regions) { if (region.type === 'file' && region.name.endsWith('.pmtiles')) { // Strip .pmtiles and date suffix (e.g. "alaska_2025-12" -> "alaska") for stable source names @@ -489,12 +596,206 @@ export class MapService implements IMapService { } } + async listCountries(): Promise { + return CountriesService.getInstance().list() + } + + async listCountryGroups(): Promise { + return CountriesService.getInstance().listGroups() + } + + async extractPreflight(params: { + countries: CountryCode[] + maxzoom?: number + }): Promise { + this.validateMaxzoom(params.maxzoom) + const countries = await CountriesService.getInstance().resolveCodes(params.countries) + const regionFilepath = await CountriesService.getInstance().writeRegionFile(countries) + const info = await this.getGlobalMapInfo() + return this.runDryRun(info, regionFilepath, params.maxzoom) + } + + private async runDryRun( + info: { url: string; date: string; key: string }, + regionFilepath: string, + maxzoom?: number + ): Promise { + const dryRunOutput = join(tmpdir(), `pmtiles-dry-run-${randomBytes(6).toString('hex')}.pmtiles`) + const args = buildPmtilesExtractArgs({ + sourceUrl: info.url, + outputFilepath: dryRunOutput, + regionFilepath, + maxzoom, + dryRun: true, + }) + + let stdout = '' + let stderr = '' + try { + const result = await execFileAsync(PMTILES_BINARY_PATH, args, { + timeout: DRY_RUN_TIMEOUT_MS, + maxBuffer: DRY_RUN_MAX_BUFFER, + }) + stdout = result.stdout + stderr = result.stderr + } catch (err: any) { + throw new Error( + `pmtiles extract --dry-run failed: ${err.message}. stderr: ${err.stderr ?? ''}` + ) + } + + const parsed = this.parseDryRunOutput(stdout + '\n' + stderr) + + return { + tiles: parsed.tiles, + bytes: parsed.bytes, + source: { url: info.url, date: info.date, key: info.key }, + } + } + + async extractRegion(params: { + countries: CountryCode[] + maxzoom?: number + label?: string + estimatedBytes?: number + }): Promise<{ filename: string; jobId?: string }> { + this.validateMaxzoom(params.maxzoom) + const countriesService = CountriesService.getInstance() + const countries = await countriesService.resolveCodes(params.countries) + const regionFilepath = await countriesService.writeRegionFile(countries) + const maxzoom = params.maxzoom ?? EXTRACT_DEFAULT_MAX_ZOOM + + const [baseAssetsExist, info, groups] = await Promise.all([ + this.ensureBaseAssets(), + this.getGlobalMapInfo(), + countriesService.listGroups(), + ]) + if (!baseAssetsExist) { + throw new Error( + 'Base map assets are missing and could not be downloaded. Please check your connection and try again.' + ) + } + + const groupMatch = findExactGroupMatch(countries, groups) + const slug = this.buildRegionSlug(countries, groupMatch) + const dateSlug = info.key.replace('.pmtiles', '') + const filename = `${slug}_${dateSlug}_z${maxzoom}.pmtiles` + const basePath = resolve(join(this.baseDirPath, 'pmtiles')) + const filepath = resolve(join(basePath, filename)) + + if (!filepath.startsWith(basePath + sep)) { + throw new Error('Invalid filename') + } + + let estimatedBytes = params.estimatedBytes ?? 0 + if (estimatedBytes === 0) { + try { + const preflight = await this.runDryRun(info, regionFilepath, maxzoom) + estimatedBytes = preflight.bytes + } catch (err) { + logger.warn(`[MapService] extractRegion preflight failed, proceeding without estimate: ${err}`) + } + } + + const title = params.label ?? this.buildRegionTitle(countries, groupMatch) + + const result = await RunExtractPmtilesJob.dispatch({ + sourceUrl: info.url, + outputFilepath: filepath, + regionFilepath, + maxzoom, + estimatedBytes, + filetype: 'map', + title, + resourceMetadata: { + resource_id: slug, + version: dateSlug, + collection_ref: null, + }, + }) + + if (!result.job) { + throw new Error('Failed to dispatch extract job') + } + + logger.info( + `[MapService] Dispatched extract job ${result.job.id} for ${filename} ` + + `(countries=[${countries.join(',')}] maxzoom=${maxzoom} est=${estimatedBytes} bytes)` + ) + + return { + filename, + jobId: result.job.id, + } + } + + private buildRegionSlug(countries: CountryCode[], groupMatch: CountryGroup | null): string { + if (groupMatch) return groupMatch.id + if (countries.length === 1) return countries[0].toLowerCase() + const hash = createHash('sha1').update(countries.join(',')).digest('hex').slice(0, 8) + return `custom-${hash}` + } + + private buildRegionTitle(countries: CountryCode[], groupMatch: CountryGroup | null): string { + if (groupMatch) return groupMatch.name + if (countries.length === 1) return countries[0] + if (countries.length <= 3) return countries.join(', ') + return `${countries.slice(0, 2).join(', ')} +${countries.length - 2} more` + } + + private validateMaxzoom(maxzoom: number | undefined): void { + if (typeof maxzoom !== 'number') return + if ( + !Number.isInteger(maxzoom) || + maxzoom < EXTRACT_MIN_ZOOM || + maxzoom > EXTRACT_MAX_ZOOM + ) { + throw new Error( + `maxzoom must be an integer in [${EXTRACT_MIN_ZOOM}, ${EXTRACT_MAX_ZOOM}]` + ) + } + } + + // go-pmtiles output format isn't stable across versions — parse loosely and + // fall back to zeros. The extract can still proceed without an estimate. + private parseDryRunOutput(output: string): { tiles: number; bytes: number } { + let bytes = 0 + let tiles = 0 + + const byteLine = output.match(/archive\s+size\s+of\s+([\d,.]+)\s*(B|KB|MB|GB|TB|bytes?)?/i) + if (byteLine) { + const raw = parseFloat(byteLine[1].replace(/,/g, '')) + const unit = (byteLine[2] ?? 'B').toUpperCase() + const multipliers: Record = { + B: 1, + BYTE: 1, + BYTES: 1, + KB: 1_000, + MB: 1_000_000, + GB: 1_000_000_000, + TB: 1_000_000_000_000, + } + bytes = Math.round(raw * (multipliers[unit] ?? 1)) + } + + const tileLine = output.match(/(?:tiles\s+to\s+extract|tiles)[^\d]*([\d,]+)/i) + if (tileLine) { + tiles = parseInt(tileLine[1].replace(/,/g, ''), 10) || 0 + } + + return { tiles, bytes } + } + async delete(file: string): Promise { let fileName = file if (!fileName.endsWith('.pmtiles')) { fileName += '.pmtiles' } + if (fileName === WORLD_BASEMAP_FILENAME) { + throw new Error('The world basemap cannot be deleted') + } + const basePath = resolve(join(this.baseDirPath, 'pmtiles')) const fullPath = resolve(join(basePath, fileName)) @@ -573,3 +874,16 @@ export class MapService implements IMapService { return baseUrl } } + +function findExactGroupMatch( + countries: CountryCode[], + groups: CountryGroup[] +): CountryGroup | null { + return ( + groups.find( + (g) => + g.countries.length === countries.length && + g.countries.every((c, i) => c === countries[i]) + ) ?? null + ) +} diff --git a/admin/app/validators/common.ts b/admin/app/validators/common.ts index 7065d4c..ba9f107 100644 --- a/admin/app/validators/common.ts +++ b/admin/app/validators/common.ts @@ -112,3 +112,31 @@ export const applyAllContentUpdatesValidator = vine.compile( .minLength(1), }) ) + +// --- Map extract (regional pmtiles download) --- + +// ISO 3166-1 alpha-2, 2 letters. Loose regex; CountriesService.resolveCodes +// does the authoritative check against the polygon dataset. +const countryCodeSchema = vine + .string() + .trim() + .toUpperCase() + .regex(/^[A-Z]{2}$/) + +const countriesArraySchema = vine.array(countryCodeSchema).minLength(1).maxLength(300) + +export const mapExtractPreflightValidator = vine.compile( + vine.object({ + countries: countriesArraySchema.clone(), + maxzoom: vine.number().min(0).max(15).optional(), + }) +) + +export const mapExtractValidator = vine.compile( + vine.object({ + countries: countriesArraySchema.clone(), + maxzoom: vine.number().min(0).max(15).optional(), + label: vine.string().trim().minLength(1).maxLength(64).optional(), + estimatedBytes: vine.number().min(0).optional(), + }) +) diff --git a/admin/commands/queue/work.ts b/admin/commands/queue/work.ts index 31bb1cc..49890e6 100644 --- a/admin/commands/queue/work.ts +++ b/admin/commands/queue/work.ts @@ -3,6 +3,7 @@ import type { CommandOptions } from '@adonisjs/core/types/ace' import { Worker } from 'bullmq' import queueConfig from '#config/queue' import { RunDownloadJob } from '#jobs/run_download_job' +import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job' import { DownloadModelJob } from '#jobs/download_model_job' import { RunBenchmarkJob } from '#jobs/run_benchmark_job' import { EmbedFileJob } from '#jobs/embed_file_job' @@ -126,6 +127,7 @@ export default class QueueWork extends BaseCommand { const queues = new Map() handlers.set(RunDownloadJob.key, new RunDownloadJob()) + handlers.set(RunExtractPmtilesJob.key, new RunExtractPmtilesJob()) handlers.set(DownloadModelJob.key, new DownloadModelJob()) handlers.set(RunBenchmarkJob.key, new RunBenchmarkJob()) handlers.set(EmbedFileJob.key, new EmbedFileJob()) @@ -133,6 +135,7 @@ export default class QueueWork extends BaseCommand { handlers.set(CheckServiceUpdatesJob.key, new CheckServiceUpdatesJob()) queues.set(RunDownloadJob.key, RunDownloadJob.queue) + queues.set(RunExtractPmtilesJob.key, RunExtractPmtilesJob.queue) queues.set(DownloadModelJob.key, DownloadModelJob.queue) queues.set(RunBenchmarkJob.key, RunBenchmarkJob.queue) queues.set(EmbedFileJob.key, EmbedFileJob.queue) @@ -149,6 +152,9 @@ export default class QueueWork extends BaseCommand { private getConcurrencyForQueue(queueName: string): number { const concurrencyMap: Record = { [RunDownloadJob.queue]: 3, + // pmtiles extract hits the Protomaps CDN with many parallel range reads per job; + // cap concurrency at 2 so a second extract doesn't starve the first. + [RunExtractPmtilesJob.queue]: 2, [DownloadModelJob.queue]: 2, // Lower concurrency for resource-intensive model downloads [RunBenchmarkJob.queue]: 1, // Run benchmarks one at a time for accurate results [EmbedFileJob.queue]: 2, // Lower concurrency for embedding jobs, can be resource intensive diff --git a/admin/constants/map_regions.ts b/admin/constants/map_regions.ts new file mode 100644 index 0000000..75f2922 --- /dev/null +++ b/admin/constants/map_regions.ts @@ -0,0 +1,32 @@ +export const PMTILES_BINARY_PATH = '/usr/local/bin/pmtiles' + +// Clamp these so a user can't ask for nonsense that never extracts +export const EXTRACT_MIN_ZOOM = 0 +export const EXTRACT_MAX_ZOOM = 15 +export const EXTRACT_DEFAULT_MAX_ZOOM = 15 + +// Low-zoom global fallback extracted once during base-asset setup (~15 MB). Layered +// underneath regional extracts so the map isn't grey outside a region's polygon. +export const WORLD_BASEMAP_FILENAME = 'world.pmtiles' +export const WORLD_BASEMAP_MAX_ZOOM = 5 +export const WORLD_BASEMAP_SOURCE_NAME = 'world' + +export interface PmtilesExtractArgOptions { + sourceUrl: string + outputFilepath: string + regionFilepath?: string + maxzoom?: number + dryRun?: boolean + downloadThreads?: number + overfetch?: number +} + +export function buildPmtilesExtractArgs(opts: PmtilesExtractArgOptions): string[] { + const args = ['extract', opts.sourceUrl, opts.outputFilepath] + if (opts.regionFilepath) args.push(`--region=${opts.regionFilepath}`) + if (typeof opts.maxzoom === 'number') args.push(`--maxzoom=${opts.maxzoom}`) + if (opts.dryRun) args.push('--dry-run') + if (typeof opts.downloadThreads === 'number') args.push(`--download-threads=${opts.downloadThreads}`) + if (typeof opts.overfetch === 'number') args.push(`--overfetch=${opts.overfetch}`) + return args +} diff --git a/admin/inertia/components/CountryPickerModal.tsx b/admin/inertia/components/CountryPickerModal.tsx new file mode 100644 index 0000000..b490360 --- /dev/null +++ b/admin/inertia/components/CountryPickerModal.tsx @@ -0,0 +1,384 @@ +import { useEffect, useMemo, useRef, useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { IconCheck, IconSearch, IconX } from '@tabler/icons-react' +import StyledModal, { StyledModalProps } from './StyledModal' +import LoadingSpinner from './LoadingSpinner' +import api from '~/lib/api' +import { formatBytes } from '~/lib/util' +import classNames from '~/lib/classNames' +import { + EXTRACT_DEFAULT_MAX_ZOOM, + EXTRACT_MAX_ZOOM, + EXTRACT_MIN_ZOOM, +} from '../../constants/map_regions' +import type { + Country, + CountryCode, + CountryGroup, + MapExtractPreflight, +} from '../../types/maps' + +export type CountryPickerModalProps = Omit< + StyledModalProps, + | 'onConfirm' + | 'open' + | 'confirmText' + | 'cancelText' + | 'confirmVariant' + | 'children' + | 'title' + | 'large' +> & { + onDownloadStart?: () => void +} + +const CountryPickerModal: React.FC = ({ + onDownloadStart, + ...modalProps +}) => { + const [selected, setSelected] = useState>(new Set()) + const [search, setSearch] = useState('') + const [maxzoom, setMaxzoom] = useState(EXTRACT_DEFAULT_MAX_ZOOM) + const [preflight, setPreflight] = useState(null) + const [loading, setLoading] = useState(false) + const [downloading, setDownloading] = useState(false) + const [errorMessage, setErrorMessage] = useState(null) + const preflightRequestIdRef = useRef(0) + + const { data: countries = [], isLoading: countriesLoading } = useQuery({ + queryKey: ['maps-countries'], + queryFn: () => api.listCountries(), + staleTime: Infinity, + }) + + const { data: groups = [] } = useQuery({ + queryKey: ['maps-country-groups'], + queryFn: () => api.listCountryGroups(), + staleTime: Infinity, + }) + + const grouped = useMemo(() => { + const q = search.trim().toLowerCase() + const filtered = q + ? countries.filter( + (c) => c.name.toLowerCase().includes(q) || c.code.toLowerCase().includes(q) + ) + : countries + + const buckets: Record = {} + for (const country of filtered) { + if (!buckets[country.continent]) buckets[country.continent] = [] + buckets[country.continent].push(country) + } + return Object.entries(buckets).sort(([a], [b]) => a.localeCompare(b)) + }, [countries, search]) + + const selectedCountries = useMemo( + () => countries.filter((c) => selected.has(c.code)), + [countries, selected] + ) + + function toggleCountry(code: CountryCode) { + setSelected((prev) => { + const next = new Set(prev) + if (next.has(code)) next.delete(code) + else next.add(code) + return next + }) + } + + function toggleGroup(group: CountryGroup) { + setSelected((prev) => { + const next = new Set(prev) + const allIn = group.countries.every((c) => next.has(c)) + if (allIn) { + group.countries.forEach((c) => next.delete(c)) + } else { + group.countries.forEach((c) => next.add(c)) + } + return next + }) + } + + function clearAll() { + setSelected(new Set()) + } + + // Auto-refresh the preflight whenever selection or maxzoom changes. Debounced + // so rapid multi-select clicks collapse into a single CDN round-trip, and + // stale-safe via requestId so an earlier slow response can't clobber a later one. + useEffect(() => { + if (selected.size === 0) { + setPreflight(null) + setErrorMessage(null) + setLoading(false) + preflightRequestIdRef.current++ + return + } + + const requestId = ++preflightRequestIdRef.current + setLoading(true) + setErrorMessage(null) + const timer = setTimeout(async () => { + try { + const res = await api.extractMapPreflight({ + countries: [...selected], + maxzoom, + }) + if (requestId !== preflightRequestIdRef.current) return + if (!res) throw new Error('Preflight returned no data') + setPreflight(res) + } catch (err: any) { + if (requestId !== preflightRequestIdRef.current) return + console.error('Preflight failed:', err) + setErrorMessage(err?.message ?? 'Estimate failed') + } finally { + if (requestId === preflightRequestIdRef.current) setLoading(false) + } + }, 400) + + return () => clearTimeout(timer) + }, [selected, maxzoom]) + + async function startDownload() { + if (selected.size === 0) { + setErrorMessage('Pick at least one country before downloading.') + return + } + if (loading || !preflight) { + setErrorMessage('Still estimating size — hold on a moment.') + return + } + try { + setDownloading(true) + setErrorMessage(null) + await api.extractMapRegion({ + countries: [...selected], + maxzoom, + estimatedBytes: preflight?.bytes, + }) + onDownloadStart?.() + } catch (err: any) { + console.error('Extract dispatch failed:', err) + setErrorMessage(err?.message ?? 'Download failed') + } finally { + setDownloading(false) + } + } + + return ( + +
+
+
+ + setSearch(e.target.value)} + placeholder={`Search ${countries.length} countries...`} + className="w-full pl-9 pr-3 py-2 rounded-md border border-border-default bg-surface-primary text-text-primary text-sm focus:outline-none focus:ring-2 focus:ring-desert-green" + /> +
+ {selected.size > 0 && ( + + )} +
+ + {groups.length > 0 && ( +
+

+ Quick picks +

+
+ {groups.map((group) => { + const allIn = + group.countries.length > 0 && + group.countries.every((c) => selected.has(c)) + return ( + + ) + })} +
+
+ )} + +
+ {countriesLoading ? ( +
+ +
+ ) : grouped.length === 0 ? ( +

+ No countries match "{search}". +

+ ) : ( + grouped.map(([continent, list]) => ( +
+
+ {continent} +
+
    + {list.map((country) => { + const isSelected = selected.has(country.code) + return ( +
  • + +
    • + ) + })} +
+
+ )) + )} +
+ + {selectedCountries.length > 0 && ( +
+

+ {selectedCountries.length} selected +

+
+ {selectedCountries.map((country) => ( + + {country.name} + + + ))} +
+
+ )} + +
+ + setMaxzoom(parseInt(e.target.value, 10))} + className="w-full accent-desert-green" + disabled={loading || downloading} + /> +
+ z{EXTRACT_MIN_ZOOM} (world) + z{EXTRACT_MAX_ZOOM} (street) +
+

+ Lower zoom = smaller file, less detail. Zoom 15 shows individual streets; + zoom 10 shows city-level detail. +

+
+ +
+ 0} + /> +
+ +
+ + ) +} + +type PreflightStatusProps = { + errorMessage: string | null + loading: boolean + preflight: MapExtractPreflight | null + hasSelection: boolean +} + +function PreflightStatus({ errorMessage, loading, preflight, hasSelection }: PreflightStatusProps) { + if (errorMessage) { + return

{errorMessage}

+ } + if (loading) { + return

Estimating size…

+ } + if (preflight) { + return ( +

+ {preflight.tiles.toLocaleString()} tiles, ~{formatBytes(preflight.bytes, 1)}{' '} + (source build {preflight.source.date}) +

+ ) + } + if (!hasSelection) { + return

Pick at least one country to estimate size.

+ } + return

Estimating size…

+} + +export default CountryPickerModal diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 2449259..77e0515 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -4,6 +4,7 @@ import { ServiceSlim } from '../../types/services' import { FileEntry } from '../../types/files' import { CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus } from '../../types/system' import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads' +import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps' import { EmbedJobWithProgress } from '../../types/rag' import type { CategoryWithStatus, CollectionWithStatus, ContentUpdateCheckResult, ResourceUpdateInfo } from '../../types/collections' import { catchInternal } from './util' @@ -557,6 +558,46 @@ class API { })() } + async listCountries() { + return catchInternal(async () => { + const response = await this.client.get<{ countries: Country[] }>('/maps/countries') + return response.data.countries + })() + } + + async listCountryGroups() { + return catchInternal(async () => { + const response = await this.client.get<{ groups: CountryGroup[] }>('/maps/country-groups') + return response.data.groups + })() + } + + async extractMapPreflight(params: { countries: CountryCode[]; maxzoom?: number }) { + return catchInternal(async () => { + const response = await this.client.post( + '/maps/extract-preflight', + params + ) + return response.data + })() + } + + async extractMapRegion(params: { + countries: CountryCode[] + maxzoom?: number + label?: string + estimatedBytes?: number + }) { + return catchInternal(async () => { + const response = await this.client.post<{ + message: string + filename: string + jobId?: string + }>('/maps/extract', params) + return response.data + })() + } + async listCuratedMapCollections() { return catchInternal(async () => { const response = await this.client.get( diff --git a/admin/inertia/pages/settings/maps.tsx b/admin/inertia/pages/settings/maps.tsx index 47055fa..dcd945f 100644 --- a/admin/inertia/pages/settings/maps.tsx +++ b/admin/inertia/pages/settings/maps.tsx @@ -13,6 +13,7 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import useDownloads from '~/hooks/useDownloads' import StyledSectionHeader from '~/components/StyledSectionHeader' import CuratedCollectionCard from '~/components/CuratedCollectionCard' +import CountryPickerModal from '~/components/CountryPickerModal' import type { CollectionWithStatus } from '../../../types/collections' import ActiveDownloads from '~/components/ActiveDownloads' import Alert from '~/components/Alert' @@ -221,6 +222,23 @@ export default function MapsManager(props: { ) } + function openCountryPickerModal() { + openModal( + { + invalidateDownloads() + addNotification({ + type: 'success', + message: 'Download queued. Watch progress below.', + }) + closeAllModals() + }} + />, + 'country-picker-modal' + ) + } + async function openDownloadModal() { openModal( )} + +