feat(Maps): regional map downloads via go-pmtiles extract (#780)
* feat(maps): add regional map downloads via go-pmtiles extract * address Copilot review feedback on PR #780 - auto-refresh preflight on selection/maxzoom change with 400ms debounce and requestId stale-safety so the confirm button no longer requires a two-step "Estimate Size" -> "Start Download" dance - safeUpdateProgress helper replaces fire-and-forget updateProgress().catch() pattern so cancelled-job errors (code -1) can't surface as unhandled rejections - gate world basemap source on worldBasemapReady - when ensureWorldBasemap() fails we already delete world.pmtiles, so emitting the source was producing 404s on every tile request - verify go-pmtiles binary SHA256 at image build time; upstream doesn't ship a checksums file so per-arch hashes are pinned as build args with a regenerate note when bumping PMTILES_VERSION
This commit is contained in:
0xGlitch
Jake Turner
parent
299b767e63
commit
94059b0aaf
25
Dockerfile
25
Dockerfile
|
|
@ -34,6 +34,31 @@ FROM base
|
|||
ARG VERSION=dev
|
||||
ARG BUILD_DATE
|
||||
ARG VCS_REF
|
||||
ARG TARGETARCH
|
||||
|
||||
# go-pmtiles (regional map extracts). Pinned so the CLI's stdout format stays
|
||||
# in sync with parseDryRunOutput().
|
||||
ARG PMTILES_VERSION=1.30.2
|
||||
# Upstream releases don't ship a checksums file, so pin per-arch SHA256 here.
|
||||
# When bumping PMTILES_VERSION, regenerate these with:
|
||||
# curl -fsSL <release-url> | sha256sum
|
||||
ARG PMTILES_SHA256_AMD64=2cd3aa18868297fc88425038f794efdc0995e0275f4ca16fa496dd79e245a40c
|
||||
ARG PMTILES_SHA256_ARM64=804cdf071834e1156af554c1a26cc42b56b9cde5a2db9c6e3653d16fb846d5fa
|
||||
RUN set -eux; \
|
||||
case "${TARGETARCH:-amd64}" in \
|
||||
amd64) PMTILES_ARCH=x86_64; PMTILES_SHA256="${PMTILES_SHA256_AMD64}" ;; \
|
||||
arm64) PMTILES_ARCH=arm64; PMTILES_SHA256="${PMTILES_SHA256_ARM64}" ;; \
|
||||
*) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
|
||||
esac; \
|
||||
TARBALL="go-pmtiles_${PMTILES_VERSION}_Linux_${PMTILES_ARCH}.tar.gz"; \
|
||||
cd /tmp; \
|
||||
curl -fsSL -o "$TARBALL" \
|
||||
"https://github.com/protomaps/go-pmtiles/releases/download/v${PMTILES_VERSION}/${TARBALL}"; \
|
||||
echo "${PMTILES_SHA256} ${TARBALL}" | sha256sum -c -; \
|
||||
tar -xzf "$TARBALL" -C /usr/local/bin pmtiles; \
|
||||
rm -f "$TARBALL"; \
|
||||
chmod +x /usr/local/bin/pmtiles; \
|
||||
/usr/local/bin/pmtiles version
|
||||
|
||||
# Labels
|
||||
LABEL org.opencontainers.image.title="Project N.O.M.A.D" \
|
||||
|
|
|
|||
|
|
@ -107,6 +107,10 @@ export default defineConfig({
|
|||
pattern: 'resources/views/**/*.edge',
|
||||
reloadServer: false,
|
||||
},
|
||||
{
|
||||
pattern: 'resources/geodata/**/*.geojson',
|
||||
reloadServer: false,
|
||||
},
|
||||
{
|
||||
pattern: 'public/**',
|
||||
reloadServer: false,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ import {
|
|||
assertNotPrivateUrl,
|
||||
downloadCollectionValidator,
|
||||
filenameParamValidator,
|
||||
mapExtractPreflightValidator,
|
||||
mapExtractValidator,
|
||||
remoteDownloadValidator,
|
||||
remoteDownloadValidatorOptional,
|
||||
} from '#validators/common'
|
||||
|
|
@ -87,6 +89,28 @@ export default class MapsController {
|
|||
}
|
||||
}
|
||||
|
||||
async listCountries({}: HttpContext) {
|
||||
return { countries: await this.mapService.listCountries() }
|
||||
}
|
||||
|
||||
async listCountryGroups({}: HttpContext) {
|
||||
return { groups: await this.mapService.listCountryGroups() }
|
||||
}
|
||||
|
||||
async extractPreflight({ request }: HttpContext) {
|
||||
const payload = await request.validateUsing(mapExtractPreflightValidator)
|
||||
return await this.mapService.extractPreflight(payload)
|
||||
}
|
||||
|
||||
async extractRegion({ request }: HttpContext) {
|
||||
const payload = await request.validateUsing(mapExtractValidator)
|
||||
const result = await this.mapService.extractRegion(payload)
|
||||
return {
|
||||
message: 'Extract started successfully',
|
||||
...result,
|
||||
}
|
||||
}
|
||||
|
||||
async styles({ request, response }: HttpContext) {
|
||||
// Automatically ensure base assets are present before generating styles
|
||||
const baseAssetsExist = await this.mapService.ensureBaseAssets()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,294 @@
|
|||
import { Job, UnrecoverableError } from 'bullmq'
|
||||
import { spawn, ChildProcess } from 'child_process'
|
||||
import { createHash } from 'crypto'
|
||||
import { readdir, stat } from 'fs/promises'
|
||||
import { basename, dirname, join } from 'path'
|
||||
import { QueueService } from '#services/queue_service'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { DownloadProgressData } from '../../types/downloads.js'
|
||||
import { PMTILES_BINARY_PATH, buildPmtilesExtractArgs } from '../../constants/map_regions.js'
|
||||
import { deleteFileIfExists } from '../utils/fs.js'
|
||||
|
||||
export interface RunExtractPmtilesJobParams {
|
||||
sourceUrl: string
|
||||
outputFilepath: string
|
||||
/** Path to a GeoJSON FeatureCollection file passed to `pmtiles extract --region`. */
|
||||
regionFilepath: string
|
||||
maxzoom?: number
|
||||
/** Hint for progress reporting; obtained from `pmtiles extract --dry-run` preflight */
|
||||
estimatedBytes?: number
|
||||
filetype: 'map'
|
||||
title?: string
|
||||
resourceMetadata?: {
|
||||
resource_id: string
|
||||
version: string
|
||||
collection_ref: string | null
|
||||
}
|
||||
}
|
||||
|
||||
export class RunExtractPmtilesJob {
|
||||
static get queue() {
|
||||
return 'pmtiles-extract'
|
||||
}
|
||||
|
||||
static get key() {
|
||||
return 'run-pmtiles-extract'
|
||||
}
|
||||
|
||||
/** In-memory registry of active child processes so in-process cancels can SIGTERM them */
|
||||
static childProcesses: Map<string, ChildProcess> = new Map()
|
||||
|
||||
static getJobId(sourceUrl: string, regionFilepath: string, maxzoom?: number): string {
|
||||
const payload = JSON.stringify({ sourceUrl, regionFilepath, maxzoom: maxzoom ?? null })
|
||||
return createHash('sha256').update(payload).digest('hex').slice(0, 16)
|
||||
}
|
||||
|
||||
/** Redis key used to signal cancellation across processes */
|
||||
static cancelKey(jobId: string): string {
|
||||
return `nomad:download:pmtiles-cancel:${jobId}`
|
||||
}
|
||||
|
||||
static async signalCancel(jobId: string): Promise<void> {
|
||||
const queueService = new QueueService()
|
||||
const queue = queueService.getQueue(this.queue)
|
||||
const client = await queue.client
|
||||
await client.set(this.cancelKey(jobId), '1', 'EX', 300)
|
||||
}
|
||||
|
||||
/** Awaits job.updateProgress and swallows BullMQ stale-job errors (code -1),
|
||||
* which occur when the job was removed from Redis (e.g. cancelled) between
|
||||
* the await being issued and the Redis write completing. Anything else
|
||||
* re-throws so it's caught by the surrounding try rather than becoming an
|
||||
* unhandled rejection. */
|
||||
private async safeUpdateProgress(job: Job, progress: DownloadProgressData): Promise<void> {
|
||||
try {
|
||||
await job.updateProgress(progress)
|
||||
} catch (err: any) {
|
||||
if (err?.code !== -1) throw err
|
||||
}
|
||||
}
|
||||
|
||||
async handle(job: Job) {
|
||||
const params = job.data as RunExtractPmtilesJobParams
|
||||
const { sourceUrl, outputFilepath, regionFilepath, maxzoom, estimatedBytes } = params
|
||||
|
||||
logger.info(
|
||||
`[RunExtractPmtilesJob] Starting extract: source=${sourceUrl} region=${regionFilepath} ` +
|
||||
`maxzoom=${maxzoom ?? 'source-max'} out=${outputFilepath}`
|
||||
)
|
||||
|
||||
const queueService = new QueueService()
|
||||
const cancelRedis = await queueService.getQueue(RunExtractPmtilesJob.queue).client
|
||||
|
||||
let userCancelled = false
|
||||
let proc: ChildProcess | null = null
|
||||
let lastReportedBytes = -1
|
||||
|
||||
// One 2s tick polls the Redis cancel signal and reads file-size for progress. pmtiles
|
||||
// writes incrementally but rewrites directories near the end so progress isn't strictly
|
||||
// monotonic — we cap at 99% and skip emit when bytes are unchanged to avoid Redis chatter.
|
||||
const tick = setInterval(async () => {
|
||||
try {
|
||||
const val = await cancelRedis.get(RunExtractPmtilesJob.cancelKey(job.id!))
|
||||
if (val) {
|
||||
await cancelRedis.del(RunExtractPmtilesJob.cancelKey(job.id!))
|
||||
userCancelled = true
|
||||
proc?.kill('SIGTERM')
|
||||
}
|
||||
} catch {
|
||||
// Redis errors non-fatal — in-memory handle also covers same-process cancels
|
||||
}
|
||||
|
||||
try {
|
||||
const fileStat = await stat(outputFilepath)
|
||||
const downloadedBytes = Number(fileStat.size)
|
||||
if (downloadedBytes === lastReportedBytes) return
|
||||
lastReportedBytes = downloadedBytes
|
||||
|
||||
const totalBytes = estimatedBytes ?? 0
|
||||
const percent =
|
||||
totalBytes > 0 ? Math.min(99, Math.floor((downloadedBytes / totalBytes) * 100)) : 0
|
||||
|
||||
await this.safeUpdateProgress(job, {
|
||||
percent,
|
||||
downloadedBytes,
|
||||
totalBytes,
|
||||
lastProgressTime: Date.now(),
|
||||
} as DownloadProgressData)
|
||||
} catch {
|
||||
// File doesn't exist yet (subprocess still setting up)
|
||||
}
|
||||
}, 2000)
|
||||
|
||||
try {
|
||||
const args = buildPmtilesExtractArgs({
|
||||
sourceUrl,
|
||||
outputFilepath,
|
||||
regionFilepath,
|
||||
maxzoom,
|
||||
downloadThreads: 8,
|
||||
overfetch: 0.2,
|
||||
})
|
||||
proc = spawn(PMTILES_BINARY_PATH, args, { stdio: ['ignore', 'pipe', 'pipe'] })
|
||||
RunExtractPmtilesJob.childProcesses.set(job.id!, proc)
|
||||
|
||||
proc.stdout?.on('data', (chunk) => {
|
||||
logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`)
|
||||
})
|
||||
proc.stderr?.on('data', (chunk) => {
|
||||
logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`)
|
||||
})
|
||||
|
||||
const exitCode: number = await new Promise((resolve, reject) => {
|
||||
proc!.on('close', (code) => resolve(code ?? -1))
|
||||
proc!.on('error', (err) => reject(err))
|
||||
})
|
||||
|
||||
if (exitCode !== 0) {
|
||||
await deleteFileIfExists(outputFilepath)
|
||||
if (userCancelled) {
|
||||
throw new UnrecoverableError(`Extract cancelled by user (exit ${exitCode})`)
|
||||
}
|
||||
throw new Error(`pmtiles extract exited with code ${exitCode}`)
|
||||
}
|
||||
|
||||
// Final progress bump — tick caps at 99 so the UI doesn't flicker to 100 mid-extract
|
||||
const finalStat = await stat(outputFilepath)
|
||||
await this.safeUpdateProgress(job, {
|
||||
percent: 100,
|
||||
downloadedBytes: Number(finalStat.size),
|
||||
totalBytes: estimatedBytes ?? Number(finalStat.size),
|
||||
lastProgressTime: Date.now(),
|
||||
} as DownloadProgressData)
|
||||
|
||||
// Reuse the HTTP download path's post-download hook so the file is registered and
|
||||
// the previous version (if any) is deleted
|
||||
await this.onComplete(params)
|
||||
|
||||
logger.info(
|
||||
`[RunExtractPmtilesJob] Completed extract: out=${outputFilepath} size=${finalStat.size} bytes`
|
||||
)
|
||||
|
||||
return { sourceUrl, outputFilepath }
|
||||
} catch (error: any) {
|
||||
if (userCancelled && !(error instanceof UnrecoverableError)) {
|
||||
throw new UnrecoverableError(`Extract cancelled: ${error.message ?? error}`)
|
||||
}
|
||||
throw error
|
||||
} finally {
|
||||
clearInterval(tick)
|
||||
RunExtractPmtilesJob.childProcesses.delete(job.id!)
|
||||
}
|
||||
}
|
||||
|
||||
private async onComplete(params: RunExtractPmtilesJobParams) {
|
||||
if (!params.resourceMetadata) return
|
||||
|
||||
const [{ default: InstalledResource }, { DateTime }, fsUtils] = await Promise.all([
|
||||
import('#models/installed_resource'),
|
||||
import('luxon'),
|
||||
import('../utils/fs.js'),
|
||||
])
|
||||
|
||||
const fileStat = await fsUtils.getFileStatsIfExists(params.outputFilepath)
|
||||
|
||||
const existing = await InstalledResource.query()
|
||||
.where('resource_id', params.resourceMetadata.resource_id)
|
||||
.where('resource_type', 'map')
|
||||
.first()
|
||||
const oldFilePath = existing?.file_path ?? null
|
||||
|
||||
await InstalledResource.updateOrCreate(
|
||||
{
|
||||
resource_id: params.resourceMetadata.resource_id,
|
||||
resource_type: 'map',
|
||||
},
|
||||
{
|
||||
version: params.resourceMetadata.version,
|
||||
collection_ref: params.resourceMetadata.collection_ref,
|
||||
url: params.sourceUrl,
|
||||
file_path: params.outputFilepath,
|
||||
file_size_bytes: fileStat ? Number(fileStat.size) : null,
|
||||
installed_at: DateTime.now(),
|
||||
}
|
||||
)
|
||||
|
||||
if (oldFilePath && oldFilePath !== params.outputFilepath) {
|
||||
try {
|
||||
await fsUtils.deleteFileIfExists(oldFilePath)
|
||||
} catch (err) {
|
||||
logger.warn(`[RunExtractPmtilesJob] Failed to delete old file ${oldFilePath}: ${err}`)
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: scan the pmtiles dir for orphans with the same resource_id that the DB
|
||||
// lookup above didn't catch — e.g. a prior extract crashed before writing its
|
||||
// InstalledResource row, or an earlier bug wrote a file without registering it.
|
||||
// Matches both curated (`<id>_YYYY-MM.pmtiles`) and regional (`<id>_YYYYMMDD_zN.pmtiles`)
|
||||
// naming — prefix-only so new filename formats don't silently miss.
|
||||
const dir = dirname(params.outputFilepath)
|
||||
const keepName = basename(params.outputFilepath)
|
||||
const prefix = `${params.resourceMetadata.resource_id}_`
|
||||
try {
|
||||
const entries = await readdir(dir)
|
||||
for (const entry of entries) {
|
||||
if (entry === keepName || !entry.endsWith('.pmtiles')) continue
|
||||
if (!entry.startsWith(prefix)) continue
|
||||
const orphanPath = join(dir, entry)
|
||||
if (orphanPath === oldFilePath) continue
|
||||
try {
|
||||
await fsUtils.deleteFileIfExists(orphanPath)
|
||||
logger.info(`[RunExtractPmtilesJob] Pruned orphan pmtiles ${orphanPath}`)
|
||||
} catch (err) {
|
||||
logger.warn(`[RunExtractPmtilesJob] Failed to prune orphan ${orphanPath}: ${err}`)
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(`[RunExtractPmtilesJob] Directory scan for orphans failed: ${err}`)
|
||||
}
|
||||
}
|
||||
|
||||
static async getById(jobId: string): Promise<Job | undefined> {
|
||||
const queueService = new QueueService()
|
||||
const queue = queueService.getQueue(this.queue)
|
||||
return await queue.getJob(jobId)
|
||||
}
|
||||
|
||||
static async dispatch(params: RunExtractPmtilesJobParams) {
|
||||
const queueService = new QueueService()
|
||||
const queue = queueService.getQueue(this.queue)
|
||||
const jobId = this.getJobId(params.sourceUrl, params.regionFilepath, params.maxzoom)
|
||||
|
||||
const existing = await queue.getJob(jobId)
|
||||
if (existing) {
|
||||
const state = await existing.getState()
|
||||
if (state === 'active' || state === 'waiting' || state === 'delayed') {
|
||||
return {
|
||||
job: existing,
|
||||
created: false,
|
||||
message: `Extract job already exists for these params`,
|
||||
}
|
||||
}
|
||||
// Stale (completed/failed) — remove so we can re-dispatch under the same deterministic id
|
||||
try {
|
||||
await existing.remove()
|
||||
} catch {
|
||||
// Already gone or locked — add() below will still report a meaningful error
|
||||
}
|
||||
}
|
||||
|
||||
// Fewer attempts than HTTP downloads — a failed extract usually means the source URL
|
||||
// rotated or the CDN is throttling, and resuming mid-extract isn't supported by the CLI
|
||||
const job = await queue.add(this.key, params, {
|
||||
jobId,
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential', delay: 60000 },
|
||||
removeOnComplete: true,
|
||||
})
|
||||
return {
|
||||
job,
|
||||
created: true,
|
||||
message: `Dispatched pmtiles extract job`,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,308 @@
|
|||
import { access, readFile, writeFile, mkdir } from 'fs/promises'
|
||||
import { join, resolve } from 'path'
|
||||
import { createHash } from 'crypto'
|
||||
import { tmpdir } from 'os'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import type { Country, CountryCode, CountryGroup } from '../../types/maps.js'
|
||||
|
||||
interface NEFeature {
|
||||
type: 'Feature'
|
||||
properties: Record<string, any>
|
||||
geometry: unknown
|
||||
}
|
||||
|
||||
interface NEFeatureCollection {
|
||||
type: 'FeatureCollection'
|
||||
features: NEFeature[]
|
||||
}
|
||||
|
||||
const COUNTRY_GEOJSON_PATH = join(
|
||||
process.cwd(),
|
||||
'resources',
|
||||
'geodata',
|
||||
'ne_50m_admin_0_countries.geojson'
|
||||
)
|
||||
|
||||
// Natural Earth country polygons are land-only (no territorial waters), so a
|
||||
// strict intersect leaves tiles fully over the ocean out of the extract —
|
||||
// coastal cities render as grey off their coast. Inflate each polygon outward
|
||||
// by ~11 km to pull in adjacent tiles without ballooning the extract size.
|
||||
const REGION_BUFFER_DEGREES = 0.1
|
||||
|
||||
const GROUP_ORDER = [
|
||||
'north-america',
|
||||
'south-america',
|
||||
'europe',
|
||||
'africa',
|
||||
'asia',
|
||||
'oceania',
|
||||
]
|
||||
|
||||
const GROUP_META: Record<string, { id: string; name: string; description: string }> = {
|
||||
'North America': {
|
||||
id: 'north-america',
|
||||
name: 'North America',
|
||||
description: 'All countries in North America and the Caribbean.',
|
||||
},
|
||||
'South America': {
|
||||
id: 'south-america',
|
||||
name: 'South America',
|
||||
description: 'All countries in South America.',
|
||||
},
|
||||
Europe: {
|
||||
id: 'europe',
|
||||
name: 'Europe',
|
||||
description: 'All countries in Europe.',
|
||||
},
|
||||
Africa: {
|
||||
id: 'africa',
|
||||
name: 'Africa',
|
||||
description: 'All countries in Africa.',
|
||||
},
|
||||
Asia: {
|
||||
id: 'asia',
|
||||
name: 'Asia',
|
||||
description: 'All countries in Asia.',
|
||||
},
|
||||
Oceania: {
|
||||
id: 'oceania',
|
||||
name: 'Oceania',
|
||||
description: 'Australia, New Zealand, and Pacific island nations.',
|
||||
},
|
||||
}
|
||||
|
||||
export class CountriesService {
|
||||
private static instance: CountriesService | null = null
|
||||
private loadPromise: Promise<void> | null = null
|
||||
private countries: Country[] = []
|
||||
private byCode: Map<CountryCode, { country: Country; feature: NEFeature }> = new Map()
|
||||
private groups: CountryGroup[] = []
|
||||
|
||||
static getInstance(): CountriesService {
|
||||
if (!this.instance) {
|
||||
this.instance = new CountriesService()
|
||||
}
|
||||
return this.instance
|
||||
}
|
||||
|
||||
private async ensureLoaded(): Promise<void> {
|
||||
if (this.byCode.size > 0) return
|
||||
if (!this.loadPromise) {
|
||||
this.loadPromise = this.load()
|
||||
}
|
||||
await this.loadPromise
|
||||
}
|
||||
|
||||
private async load(): Promise<void> {
|
||||
const raw = await readFile(COUNTRY_GEOJSON_PATH, 'utf8')
|
||||
const fc = JSON.parse(raw) as NEFeatureCollection
|
||||
|
||||
// Natural Earth reuses a sovereign state's ISO_A2 for its dependencies
|
||||
// (e.g. AU covers both Australia and Australian territories). Sort so the
|
||||
// sovereign mainland wins the ISO-code slot, and skip any subsequent
|
||||
// same-code dependency — otherwise the "AU" entry ends up being some tiny
|
||||
// island territory.
|
||||
const sortedFeatures = [...fc.features].sort((a, b) => typeRank(a) - typeRank(b))
|
||||
|
||||
const countries: Country[] = []
|
||||
const byCode = new Map<CountryCode, { country: Country; feature: NEFeature }>()
|
||||
const groupCodes: Record<string, CountryCode[]> = {}
|
||||
|
||||
for (const feature of sortedFeatures) {
|
||||
const p = feature.properties
|
||||
const code = resolveIso2(p)
|
||||
if (!code) continue
|
||||
if (byCode.has(code)) continue
|
||||
|
||||
const continent = typeof p.CONTINENT === 'string' ? p.CONTINENT : 'Other'
|
||||
if (continent === 'Antarctica' || continent === 'Seven seas (open ocean)') continue
|
||||
|
||||
const country: Country = {
|
||||
code,
|
||||
code3: resolveIso3(p) ?? code,
|
||||
name: typeof p.NAME === 'string' ? p.NAME : code,
|
||||
continent,
|
||||
subregion: typeof p.SUBREGION === 'string' ? p.SUBREGION : continent,
|
||||
population: typeof p.POP_EST === 'number' ? p.POP_EST : 0,
|
||||
}
|
||||
|
||||
countries.push(country)
|
||||
byCode.set(code, { country, feature })
|
||||
|
||||
if (GROUP_META[continent]) {
|
||||
const groupId = GROUP_META[continent].id
|
||||
if (!groupCodes[groupId]) groupCodes[groupId] = []
|
||||
groupCodes[groupId].push(code)
|
||||
}
|
||||
}
|
||||
|
||||
countries.sort((a, b) => a.name.localeCompare(b.name))
|
||||
|
||||
const groups: CountryGroup[] = GROUP_ORDER.flatMap((groupId) => {
|
||||
const meta = Object.values(GROUP_META).find((m) => m.id === groupId)
|
||||
if (!meta) return []
|
||||
const codes = (groupCodes[groupId] ?? []).slice().sort()
|
||||
if (codes.length === 0) return []
|
||||
return [{ id: meta.id, name: meta.name, description: meta.description, countries: codes }]
|
||||
})
|
||||
|
||||
this.countries = countries
|
||||
this.byCode = byCode
|
||||
this.groups = groups
|
||||
|
||||
logger.info(
|
||||
`[CountriesService] Loaded ${countries.length} countries across ${groups.length} groups`
|
||||
)
|
||||
}
|
||||
|
||||
async list(): Promise<Country[]> {
|
||||
await this.ensureLoaded()
|
||||
return this.countries
|
||||
}
|
||||
|
||||
async listGroups(): Promise<CountryGroup[]> {
|
||||
await this.ensureLoaded()
|
||||
return this.groups
|
||||
}
|
||||
|
||||
/** Throws when a supplied code does not map to a known country. */
|
||||
async resolveCodes(codes: CountryCode[]): Promise<CountryCode[]> {
|
||||
await this.ensureLoaded()
|
||||
const normalized = [...new Set(codes.map((c) => c.toUpperCase()))].sort()
|
||||
const unknown = normalized.filter((c) => !this.byCode.has(c))
|
||||
if (unknown.length > 0) {
|
||||
throw new Error(`Unknown country code(s): ${unknown.join(', ')}`)
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
/**
|
||||
* Filename is keyed on a hash of the sorted ISO codes + buffer size so
|
||||
* repeated calls with the same selection reuse the same path, and bumping
|
||||
* the buffer auto-invalidates stale files.
|
||||
*/
|
||||
async writeRegionFile(codes: CountryCode[]): Promise<string> {
|
||||
await this.ensureLoaded()
|
||||
const resolved = await this.resolveCodes(codes)
|
||||
const key = `b${REGION_BUFFER_DEGREES}:${resolved.join(',')}`
|
||||
const hash = createHash('sha1').update(key).digest('hex').slice(0, 12)
|
||||
|
||||
const dir = resolve(tmpdir(), 'nomad-pmtiles-regions')
|
||||
await mkdir(dir, { recursive: true })
|
||||
const filepath = join(dir, `region-${hash}.geojson`)
|
||||
|
||||
try {
|
||||
await access(filepath)
|
||||
return filepath
|
||||
} catch {}
|
||||
|
||||
const fc = {
|
||||
type: 'FeatureCollection',
|
||||
features: resolved.map((code) => {
|
||||
const entry = this.byCode.get(code)!
|
||||
return {
|
||||
type: 'Feature',
|
||||
properties: { iso: code, name: entry.country.name },
|
||||
geometry: bufferGeometry(entry.feature.geometry, REGION_BUFFER_DEGREES),
|
||||
}
|
||||
}),
|
||||
}
|
||||
|
||||
await writeFile(filepath, JSON.stringify(fc))
|
||||
return filepath
|
||||
}
|
||||
}
|
||||
|
||||
function typeRank(f: NEFeature): number {
|
||||
const t = typeof f.properties.TYPE === 'string' ? f.properties.TYPE : ''
|
||||
if (t === 'Sovereign country') return 0
|
||||
if (t === 'Country') return 1
|
||||
if (t === 'Sovereignty') return 2
|
||||
if (t === 'Disputed') return 3
|
||||
if (t === 'Dependency') return 4
|
||||
return 5
|
||||
}
|
||||
|
||||
function resolveIso2(p: Record<string, any>): CountryCode | null {
|
||||
// Natural Earth's ISO_A2 sometimes holds political escapes like "CN-TW" for
|
||||
// Taiwan or "-99" for countries involved in disputes. Only accept clean
|
||||
// 2-letter codes; fall back to ISO_A2_EH (which reliably has the real code).
|
||||
const primary = typeof p.ISO_A2 === 'string' ? p.ISO_A2 : null
|
||||
if (primary && /^[A-Z]{2}$/i.test(primary)) return primary.toUpperCase()
|
||||
const fallback = typeof p.ISO_A2_EH === 'string' ? p.ISO_A2_EH : null
|
||||
if (fallback && /^[A-Z]{2}$/i.test(fallback)) return fallback.toUpperCase()
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Inflate each polygon ring outward by `buffer` degrees via per-vertex
|
||||
* averaged-normal offset. Not geodesically accurate — but at small buffers
|
||||
* (<= 0.2°) it's within a few percent of a proper geodesic buffer at
|
||||
* country scale, which is plenty for tile-inclusion purposes.
|
||||
*/
|
||||
function bufferGeometry(geometry: unknown, buffer: number): unknown {
|
||||
const geom = geometry as { type: string; coordinates: any }
|
||||
if (geom?.type === 'Polygon') {
|
||||
return { type: 'Polygon', coordinates: bufferPolygonRings(geom.coordinates, buffer) }
|
||||
}
|
||||
if (geom?.type === 'MultiPolygon') {
|
||||
return {
|
||||
type: 'MultiPolygon',
|
||||
coordinates: geom.coordinates.map((poly: number[][][]) =>
|
||||
bufferPolygonRings(poly, buffer)
|
||||
),
|
||||
}
|
||||
}
|
||||
return geometry
|
||||
}
|
||||
|
||||
function bufferPolygonRings(rings: number[][][], buffer: number): number[][][] {
|
||||
return rings.map((ring) => bufferRing(ring, buffer))
|
||||
}
|
||||
|
||||
function bufferRing(ring: number[][], buffer: number): number[][] {
|
||||
if (ring.length < 4) return ring
|
||||
const sign = signedArea(ring) > 0 ? 1 : -1
|
||||
const n = ring.length - 1
|
||||
const out: number[][] = []
|
||||
for (let i = 0; i < n; i++) {
|
||||
const prev = ring[(i - 1 + n) % n]
|
||||
const curr = ring[i]
|
||||
const next = ring[(i + 1) % n]
|
||||
const e1x = curr[0] - prev[0]
|
||||
const e1y = curr[1] - prev[1]
|
||||
const e2x = next[0] - curr[0]
|
||||
const e2y = next[1] - curr[1]
|
||||
const l1 = Math.hypot(e1x, e1y) || 1
|
||||
const l2 = Math.hypot(e2x, e2y) || 1
|
||||
const n1x = (e1y / l1) * sign
|
||||
const n1y = (-e1x / l1) * sign
|
||||
const n2x = (e2y / l2) * sign
|
||||
const n2y = (-e2x / l2) * sign
|
||||
const sumX = n1x + n2x
|
||||
const sumY = n1y + n2y
|
||||
const sl = Math.hypot(sumX, sumY) || 1
|
||||
out.push([curr[0] + (sumX / sl) * buffer, curr[1] + (sumY / sl) * buffer])
|
||||
}
|
||||
out.push(out[0])
|
||||
return out
|
||||
}
|
||||
|
||||
function signedArea(ring: number[][]): number {
|
||||
let a = 0
|
||||
for (let i = 0; i < ring.length - 1; i++) {
|
||||
a += ring[i][0] * ring[i + 1][1] - ring[i + 1][0] * ring[i][1]
|
||||
}
|
||||
return a / 2
|
||||
}
|
||||
|
||||
function resolveIso3(p: Record<string, any>): string | null {
|
||||
const primary = typeof p.ISO_A3 === 'string' ? p.ISO_A3 : null
|
||||
if (primary && primary !== '-99') return primary.toUpperCase()
|
||||
const fallback = typeof p.ISO_A3_EH === 'string' ? p.ISO_A3_EH : null
|
||||
if (fallback && fallback !== '-99') return fallback.toUpperCase()
|
||||
const adm = typeof p.ADM0_A3 === 'string' ? p.ADM0_A3 : null
|
||||
if (adm && adm !== '-99') return adm.toUpperCase()
|
||||
return null
|
||||
}
|
||||
|
||||
|
|
@ -1,13 +1,19 @@
|
|||
import { inject } from '@adonisjs/core'
|
||||
import { QueueService } from './queue_service.js'
|
||||
import { RunDownloadJob } from '#jobs/run_download_job'
|
||||
import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job'
|
||||
import type { RunExtractPmtilesJobParams } from '#jobs/run_extract_pmtiles_job'
|
||||
import { DownloadModelJob } from '#jobs/download_model_job'
|
||||
import { DownloadJobWithProgress, DownloadProgressData } from '../../types/downloads.js'
|
||||
import type { Job, Queue } from 'bullmq'
|
||||
import { normalize } from 'path'
|
||||
import { deleteFileIfExists } from '../utils/fs.js'
|
||||
import transmit from '@adonisjs/transmit/services/main'
|
||||
import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
|
||||
|
||||
type FileJobState = 'waiting' | 'active' | 'delayed' | 'failed'
|
||||
type TaggedJob = { job: Job; state: FileJobState }
|
||||
|
||||
@inject()
|
||||
export class DownloadService {
|
||||
constructor(private queueService: QueueService) {}
|
||||
|
|
@ -26,27 +32,32 @@ export class DownloadService {
|
|||
return { percent: parseInt(String(progress), 10) || 0 }
|
||||
}
|
||||
|
||||
async listDownloadJobs(filetype?: string): Promise<DownloadJobWithProgress[]> {
|
||||
// Get regular file download jobs (zim, map, etc.) — query each state separately so we can
|
||||
// tag each job with its actual BullMQ state rather than guessing from progress data.
|
||||
const queue = this.queueService.getQueue(RunDownloadJob.queue)
|
||||
type FileJobState = 'waiting' | 'active' | 'delayed' | 'failed'
|
||||
|
||||
const [waitingJobs, activeJobs, delayedJobs, failedJobs] = await Promise.all([
|
||||
/** Fetch all non-completed jobs from a queue, tagged with their current BullMQ state */
|
||||
private async fetchJobsWithStates(queueName: string): Promise<TaggedJob[]> {
|
||||
const queue = this.queueService.getQueue(queueName)
|
||||
const [waiting, active, delayed, failed] = await Promise.all([
|
||||
queue.getJobs(['waiting']),
|
||||
queue.getJobs(['active']),
|
||||
queue.getJobs(['delayed']),
|
||||
queue.getJobs(['failed']),
|
||||
])
|
||||
|
||||
const taggedFileJobs: Array<{ job: (typeof waitingJobs)[0]; state: FileJobState }> = [
|
||||
...waitingJobs.map((j) => ({ job: j, state: 'waiting' as const })),
|
||||
...activeJobs.map((j) => ({ job: j, state: 'active' as const })),
|
||||
...delayedJobs.map((j) => ({ job: j, state: 'delayed' as const })),
|
||||
...failedJobs.map((j) => ({ job: j, state: 'failed' as const })),
|
||||
return [
|
||||
...waiting.map((j) => ({ job: j, state: 'waiting' as const })),
|
||||
...active.map((j) => ({ job: j, state: 'active' as const })),
|
||||
...delayed.map((j) => ({ job: j, state: 'delayed' as const })),
|
||||
...failed.map((j) => ({ job: j, state: 'failed' as const })),
|
||||
]
|
||||
}
|
||||
|
||||
const fileDownloads = taggedFileJobs.map(({ job, state }) => {
|
||||
async listDownloadJobs(filetype?: string): Promise<DownloadJobWithProgress[]> {
|
||||
const modelQueue = this.queueService.getQueue(DownloadModelJob.queue)
|
||||
const [fileTagged, extractTagged, modelJobs] = await Promise.all([
|
||||
this.fetchJobsWithStates(RunDownloadJob.queue),
|
||||
this.fetchJobsWithStates(RunExtractPmtilesJob.queue),
|
||||
modelQueue.getJobs(['waiting', 'active', 'delayed', 'failed']),
|
||||
])
|
||||
|
||||
const fileDownloads = fileTagged.map(({ job, state }) => {
|
||||
const parsed = this.parseProgress(job.progress)
|
||||
return {
|
||||
jobId: job.id!.toString(),
|
||||
|
|
@ -63,26 +74,36 @@ export class DownloadService {
|
|||
}
|
||||
})
|
||||
|
||||
// Get Ollama model download jobs
|
||||
const modelQueue = this.queueService.getQueue(DownloadModelJob.queue)
|
||||
const modelJobs = await modelQueue.getJobs(['waiting', 'active', 'delayed', 'failed'])
|
||||
const extractDownloads = extractTagged.map(({ job, state }) => {
|
||||
const parsed = this.parseProgress(job.progress)
|
||||
return {
|
||||
jobId: job.id!.toString(),
|
||||
url: job.data.sourceUrl,
|
||||
progress: parsed.percent,
|
||||
filepath: normalize(job.data.outputFilepath),
|
||||
filetype: job.data.filetype || 'map',
|
||||
title: job.data.title || undefined,
|
||||
downloadedBytes: parsed.downloadedBytes,
|
||||
totalBytes: parsed.totalBytes || job.data.estimatedBytes || undefined,
|
||||
lastProgressTime: parsed.lastProgressTime,
|
||||
status: state,
|
||||
failedReason: job.failedReason || undefined,
|
||||
}
|
||||
})
|
||||
|
||||
const modelDownloads = modelJobs.map((job) => ({
|
||||
jobId: job.id!.toString(),
|
||||
url: job.data.modelName || 'Unknown Model', // Use model name as url
|
||||
url: job.data.modelName || 'Unknown Model',
|
||||
progress: parseInt(job.progress.toString(), 10),
|
||||
filepath: job.data.modelName || 'Unknown Model', // Use model name as filepath
|
||||
filepath: job.data.modelName || 'Unknown Model',
|
||||
filetype: 'model',
|
||||
status: (job.failedReason ? 'failed' : 'active') as 'active' | 'failed',
|
||||
failedReason: job.failedReason || undefined,
|
||||
}))
|
||||
|
||||
const allDownloads = [...fileDownloads, ...modelDownloads]
|
||||
|
||||
// Filter by filetype if specified
|
||||
const allDownloads = [...fileDownloads, ...extractDownloads, ...modelDownloads]
|
||||
const filtered = allDownloads.filter((job) => !filetype || job.filetype === filetype)
|
||||
|
||||
// Sort: active downloads first (by progress desc), then failed at the bottom
|
||||
return filtered.sort((a, b) => {
|
||||
if (a.status === 'failed' && b.status !== 'failed') return 1
|
||||
if (a.status !== 'failed' && b.status === 'failed') return -1
|
||||
|
|
@ -91,7 +112,11 @@ export class DownloadService {
|
|||
}
|
||||
|
||||
async removeFailedJob(jobId: string): Promise<void> {
|
||||
for (const queueName of [RunDownloadJob.queue, DownloadModelJob.queue]) {
|
||||
for (const queueName of [
|
||||
RunDownloadJob.queue,
|
||||
RunExtractPmtilesJob.queue,
|
||||
DownloadModelJob.queue,
|
||||
]) {
|
||||
const queue = this.queueService.getQueue(queueName)
|
||||
const job = await queue.getJob(jobId)
|
||||
if (job) {
|
||||
|
|
@ -113,7 +138,6 @@ export class DownloadService {
|
|||
}
|
||||
|
||||
async cancelJob(jobId: string): Promise<{ success: boolean; message: string }> {
|
||||
// Try the file download queue first (the original PR #554 path)
|
||||
const queue = this.queueService.getQueue(RunDownloadJob.queue)
|
||||
const job = await queue.getJob(jobId)
|
||||
|
||||
|
|
@ -121,7 +145,13 @@ export class DownloadService {
|
|||
return await this._cancelFileDownloadJob(jobId, job, queue)
|
||||
}
|
||||
|
||||
// Fall through to the model download queue
|
||||
const extractQueue = this.queueService.getQueue(RunExtractPmtilesJob.queue)
|
||||
const extractJob = await extractQueue.getJob(jobId)
|
||||
|
||||
if (extractJob) {
|
||||
return await this._cancelExtractJob(jobId, extractJob, extractQueue)
|
||||
}
|
||||
|
||||
const modelQueue = this.queueService.getQueue(DownloadModelJob.queue)
|
||||
const modelJob = await modelQueue.getJob(jobId)
|
||||
|
||||
|
|
@ -129,11 +159,37 @@ export class DownloadService {
|
|||
return await this._cancelModelDownloadJob(jobId, modelJob, modelQueue)
|
||||
}
|
||||
|
||||
// Not found in either queue
|
||||
return { success: true, message: 'Job not found (may have already completed)' }
|
||||
}
|
||||
|
||||
/** Cancel a content download (zim, map, pmtiles, etc.) — original PR #554 logic */
|
||||
private async _cancelExtractJob(
|
||||
jobId: string,
|
||||
job: Job<RunExtractPmtilesJobParams>,
|
||||
queue: Queue<RunExtractPmtilesJobParams>
|
||||
): Promise<{ success: boolean; message: string }> {
|
||||
const outputFilepath = job.data.outputFilepath
|
||||
|
||||
await RunExtractPmtilesJob.signalCancel(jobId)
|
||||
|
||||
// Same-process fallback when worker and API share a process
|
||||
RunExtractPmtilesJob.childProcesses.get(jobId)?.kill('SIGTERM')
|
||||
RunExtractPmtilesJob.childProcesses.delete(jobId)
|
||||
|
||||
await this._pollForTerminalState(job, jobId)
|
||||
await this._removeJobWithLockFallback(job, queue, RunExtractPmtilesJob.queue, jobId)
|
||||
|
||||
if (outputFilepath) {
|
||||
try {
|
||||
await deleteFileIfExists(outputFilepath)
|
||||
} catch {
|
||||
// File may not exist yet (subprocess may not have opened it)
|
||||
}
|
||||
}
|
||||
|
||||
return { success: true, message: 'Extract cancelled and partial file deleted' }
|
||||
}
|
||||
|
||||
/** Cancel a content download (zim, map, pmtiles, etc.) */
|
||||
private async _cancelFileDownloadJob(
|
||||
jobId: string,
|
||||
job: any,
|
||||
|
|
|
|||
|
|
@ -16,11 +16,35 @@ import {
|
|||
import { join, resolve, sep } from 'path'
|
||||
import urlJoin from 'url-join'
|
||||
import { RunDownloadJob } from '#jobs/run_download_job'
|
||||
import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { assertNotPrivateUrl } from '#validators/common'
|
||||
import InstalledResource from '#models/installed_resource'
|
||||
import { CollectionManifestService } from './collection_manifest_service.js'
|
||||
import type { CollectionWithStatus, MapsSpec } from '../../types/collections.js'
|
||||
import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps.js'
|
||||
import {
|
||||
EXTRACT_DEFAULT_MAX_ZOOM,
|
||||
EXTRACT_MAX_ZOOM,
|
||||
EXTRACT_MIN_ZOOM,
|
||||
PMTILES_BINARY_PATH,
|
||||
WORLD_BASEMAP_FILENAME,
|
||||
WORLD_BASEMAP_MAX_ZOOM,
|
||||
WORLD_BASEMAP_SOURCE_NAME,
|
||||
buildPmtilesExtractArgs,
|
||||
} from '../../constants/map_regions.js'
|
||||
import { CountriesService } from './countries_service.js'
|
||||
import { execFile } from 'child_process'
|
||||
import { createHash, randomBytes } from 'crypto'
|
||||
import { tmpdir } from 'os'
|
||||
import { promisify } from 'util'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const DRY_RUN_TIMEOUT_MS = 60_000
|
||||
const DRY_RUN_MAX_BUFFER = 256 * 1024
|
||||
// Real extract of z0-5 world tiles; generous to tolerate slow/metered links
|
||||
// since a failure leaves the map grey for uncovered regions.
|
||||
const WORLD_BASEMAP_EXTRACT_TIMEOUT_MS = 5 * 60_000
|
||||
|
||||
const PROTOMAPS_BUILDS_METADATA_URL = 'https://build-metadata.protomaps.dev/builds.json'
|
||||
const PROTOMAPS_BUILD_BASE_URL = 'https://build.protomaps.com'
|
||||
|
|
@ -53,10 +77,15 @@ export class MapService implements IMapService {
|
|||
private readonly baseAssetsTarFile = 'base-assets.tar.gz'
|
||||
private readonly baseDirPath = join(process.cwd(), this.mapStoragePath)
|
||||
private baseAssetsExistCache: boolean | null = null
|
||||
private worldBasemapReady = false
|
||||
private worldBasemapInFlight: Promise<void> | null = null
|
||||
|
||||
async listRegions() {
|
||||
const files = (await this.listAllMapStorageItems()).filter(
|
||||
(item) => item.type === 'file' && item.name.endsWith('.pmtiles')
|
||||
(item) =>
|
||||
item.type === 'file' &&
|
||||
item.name.endsWith('.pmtiles') &&
|
||||
item.name !== WORLD_BASEMAP_FILENAME
|
||||
)
|
||||
|
||||
return {
|
||||
|
|
@ -327,11 +356,76 @@ export class MapService implements IMapService {
|
|||
|
||||
async ensureBaseAssets(): Promise<boolean> {
|
||||
const exists = await this.checkBaseAssetsExist()
|
||||
if (exists) {
|
||||
return true
|
||||
if (!exists) {
|
||||
const downloaded = await this.downloadBaseAssets()
|
||||
if (!downloaded) return false
|
||||
}
|
||||
|
||||
return await this.downloadBaseAssets()
|
||||
try {
|
||||
await this.ensureWorldBasemap()
|
||||
} catch (err) {
|
||||
logger.warn(`[MapService] World basemap setup failed, continuing without it: ${err}`)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a low-zoom global basemap once so the map isn't grey outside a
|
||||
* regional extract's polygon. Cheap (~15 MB, a handful of HTTP range
|
||||
* requests) and layered underneath regional sources at render time.
|
||||
*
|
||||
* Memoizes success in-process, and de-duplicates concurrent callers via a
|
||||
* shared in-flight promise so two simultaneous `/maps` requests on a cold
|
||||
* start don't both launch `pmtiles extract` against the same output path.
|
||||
*/
|
||||
private async ensureWorldBasemap(): Promise<void> {
|
||||
if (this.worldBasemapReady) return
|
||||
if (this.worldBasemapInFlight) return this.worldBasemapInFlight
|
||||
this.worldBasemapInFlight = this._setupWorldBasemap().finally(() => {
|
||||
this.worldBasemapInFlight = null
|
||||
})
|
||||
return this.worldBasemapInFlight
|
||||
}
|
||||
|
||||
private async _setupWorldBasemap(): Promise<void> {
|
||||
const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
|
||||
const filepath = resolve(join(basePath, WORLD_BASEMAP_FILENAME))
|
||||
if (!filepath.startsWith(basePath + sep)) {
|
||||
throw new Error('Invalid world basemap path')
|
||||
}
|
||||
|
||||
await ensureDirectoryExists(basePath)
|
||||
|
||||
const existing = await getFileStatsIfExists(filepath)
|
||||
if (existing && Number(existing.size) > 0) {
|
||||
this.worldBasemapReady = true
|
||||
return
|
||||
}
|
||||
|
||||
const info = await this.getGlobalMapInfo()
|
||||
const args = buildPmtilesExtractArgs({
|
||||
sourceUrl: info.url,
|
||||
outputFilepath: filepath,
|
||||
maxzoom: WORLD_BASEMAP_MAX_ZOOM,
|
||||
downloadThreads: 4,
|
||||
})
|
||||
|
||||
logger.info(
|
||||
`[MapService] Extracting world basemap (z0-${WORLD_BASEMAP_MAX_ZOOM}) from ${info.url}`
|
||||
)
|
||||
try {
|
||||
await execFileAsync(PMTILES_BINARY_PATH, args, {
|
||||
timeout: WORLD_BASEMAP_EXTRACT_TIMEOUT_MS,
|
||||
maxBuffer: DRY_RUN_MAX_BUFFER,
|
||||
})
|
||||
this.worldBasemapReady = true
|
||||
} catch (err: any) {
|
||||
await deleteFileIfExists(filepath)
|
||||
throw new Error(
|
||||
`pmtiles extract for world basemap failed: ${err.message}. stderr: ${err.stderr ?? ''}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private async checkBaseAssetsExist(useCache: boolean = true): Promise<boolean> {
|
||||
|
|
@ -367,6 +461,19 @@ export class MapService implements IMapService {
|
|||
const sources: BaseStylesFile['sources'][] = []
|
||||
const baseUrl = this.getPublicFileBaseUrl(host, 'pmtiles', protocol)
|
||||
|
||||
// World basemap goes first so its layers render underneath regional extracts.
|
||||
// Only emitted when ensureWorldBasemap() succeeded — otherwise the style would
|
||||
// reference a file that doesn't exist and produce 404s on every tile request.
|
||||
if (this.worldBasemapReady) {
|
||||
const worldSource: BaseStylesFile['sources'] = {}
|
||||
worldSource[WORLD_BASEMAP_SOURCE_NAME] = {
|
||||
type: 'vector',
|
||||
attribution: PMTILES_ATTRIBUTION,
|
||||
url: `pmtiles://${urlJoin(baseUrl, WORLD_BASEMAP_FILENAME)}`,
|
||||
}
|
||||
sources.push(worldSource)
|
||||
}
|
||||
|
||||
for (const region of regions) {
|
||||
if (region.type === 'file' && region.name.endsWith('.pmtiles')) {
|
||||
// Strip .pmtiles and date suffix (e.g. "alaska_2025-12" -> "alaska") for stable source names
|
||||
|
|
@ -489,12 +596,206 @@ export class MapService implements IMapService {
|
|||
}
|
||||
}
|
||||
|
||||
async listCountries(): Promise<Country[]> {
|
||||
return CountriesService.getInstance().list()
|
||||
}
|
||||
|
||||
async listCountryGroups(): Promise<CountryGroup[]> {
|
||||
return CountriesService.getInstance().listGroups()
|
||||
}
|
||||
|
||||
async extractPreflight(params: {
|
||||
countries: CountryCode[]
|
||||
maxzoom?: number
|
||||
}): Promise<MapExtractPreflight> {
|
||||
this.validateMaxzoom(params.maxzoom)
|
||||
const countries = await CountriesService.getInstance().resolveCodes(params.countries)
|
||||
const regionFilepath = await CountriesService.getInstance().writeRegionFile(countries)
|
||||
const info = await this.getGlobalMapInfo()
|
||||
return this.runDryRun(info, regionFilepath, params.maxzoom)
|
||||
}
|
||||
|
||||
private async runDryRun(
|
||||
info: { url: string; date: string; key: string },
|
||||
regionFilepath: string,
|
||||
maxzoom?: number
|
||||
): Promise<MapExtractPreflight> {
|
||||
const dryRunOutput = join(tmpdir(), `pmtiles-dry-run-${randomBytes(6).toString('hex')}.pmtiles`)
|
||||
const args = buildPmtilesExtractArgs({
|
||||
sourceUrl: info.url,
|
||||
outputFilepath: dryRunOutput,
|
||||
regionFilepath,
|
||||
maxzoom,
|
||||
dryRun: true,
|
||||
})
|
||||
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
try {
|
||||
const result = await execFileAsync(PMTILES_BINARY_PATH, args, {
|
||||
timeout: DRY_RUN_TIMEOUT_MS,
|
||||
maxBuffer: DRY_RUN_MAX_BUFFER,
|
||||
})
|
||||
stdout = result.stdout
|
||||
stderr = result.stderr
|
||||
} catch (err: any) {
|
||||
throw new Error(
|
||||
`pmtiles extract --dry-run failed: ${err.message}. stderr: ${err.stderr ?? ''}`
|
||||
)
|
||||
}
|
||||
|
||||
const parsed = this.parseDryRunOutput(stdout + '\n' + stderr)
|
||||
|
||||
return {
|
||||
tiles: parsed.tiles,
|
||||
bytes: parsed.bytes,
|
||||
source: { url: info.url, date: info.date, key: info.key },
|
||||
}
|
||||
}
|
||||
|
||||
async extractRegion(params: {
|
||||
countries: CountryCode[]
|
||||
maxzoom?: number
|
||||
label?: string
|
||||
estimatedBytes?: number
|
||||
}): Promise<{ filename: string; jobId?: string }> {
|
||||
this.validateMaxzoom(params.maxzoom)
|
||||
const countriesService = CountriesService.getInstance()
|
||||
const countries = await countriesService.resolveCodes(params.countries)
|
||||
const regionFilepath = await countriesService.writeRegionFile(countries)
|
||||
const maxzoom = params.maxzoom ?? EXTRACT_DEFAULT_MAX_ZOOM
|
||||
|
||||
const [baseAssetsExist, info, groups] = await Promise.all([
|
||||
this.ensureBaseAssets(),
|
||||
this.getGlobalMapInfo(),
|
||||
countriesService.listGroups(),
|
||||
])
|
||||
if (!baseAssetsExist) {
|
||||
throw new Error(
|
||||
'Base map assets are missing and could not be downloaded. Please check your connection and try again.'
|
||||
)
|
||||
}
|
||||
|
||||
const groupMatch = findExactGroupMatch(countries, groups)
|
||||
const slug = this.buildRegionSlug(countries, groupMatch)
|
||||
const dateSlug = info.key.replace('.pmtiles', '')
|
||||
const filename = `${slug}_${dateSlug}_z${maxzoom}.pmtiles`
|
||||
const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
|
||||
const filepath = resolve(join(basePath, filename))
|
||||
|
||||
if (!filepath.startsWith(basePath + sep)) {
|
||||
throw new Error('Invalid filename')
|
||||
}
|
||||
|
||||
let estimatedBytes = params.estimatedBytes ?? 0
|
||||
if (estimatedBytes === 0) {
|
||||
try {
|
||||
const preflight = await this.runDryRun(info, regionFilepath, maxzoom)
|
||||
estimatedBytes = preflight.bytes
|
||||
} catch (err) {
|
||||
logger.warn(`[MapService] extractRegion preflight failed, proceeding without estimate: ${err}`)
|
||||
}
|
||||
}
|
||||
|
||||
const title = params.label ?? this.buildRegionTitle(countries, groupMatch)
|
||||
|
||||
const result = await RunExtractPmtilesJob.dispatch({
|
||||
sourceUrl: info.url,
|
||||
outputFilepath: filepath,
|
||||
regionFilepath,
|
||||
maxzoom,
|
||||
estimatedBytes,
|
||||
filetype: 'map',
|
||||
title,
|
||||
resourceMetadata: {
|
||||
resource_id: slug,
|
||||
version: dateSlug,
|
||||
collection_ref: null,
|
||||
},
|
||||
})
|
||||
|
||||
if (!result.job) {
|
||||
throw new Error('Failed to dispatch extract job')
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`[MapService] Dispatched extract job ${result.job.id} for ${filename} ` +
|
||||
`(countries=[${countries.join(',')}] maxzoom=${maxzoom} est=${estimatedBytes} bytes)`
|
||||
)
|
||||
|
||||
return {
|
||||
filename,
|
||||
jobId: result.job.id,
|
||||
}
|
||||
}
|
||||
|
||||
private buildRegionSlug(countries: CountryCode[], groupMatch: CountryGroup | null): string {
|
||||
if (groupMatch) return groupMatch.id
|
||||
if (countries.length === 1) return countries[0].toLowerCase()
|
||||
const hash = createHash('sha1').update(countries.join(',')).digest('hex').slice(0, 8)
|
||||
return `custom-${hash}`
|
||||
}
|
||||
|
||||
private buildRegionTitle(countries: CountryCode[], groupMatch: CountryGroup | null): string {
|
||||
if (groupMatch) return groupMatch.name
|
||||
if (countries.length === 1) return countries[0]
|
||||
if (countries.length <= 3) return countries.join(', ')
|
||||
return `${countries.slice(0, 2).join(', ')} +${countries.length - 2} more`
|
||||
}
|
||||
|
||||
private validateMaxzoom(maxzoom: number | undefined): void {
|
||||
if (typeof maxzoom !== 'number') return
|
||||
if (
|
||||
!Number.isInteger(maxzoom) ||
|
||||
maxzoom < EXTRACT_MIN_ZOOM ||
|
||||
maxzoom > EXTRACT_MAX_ZOOM
|
||||
) {
|
||||
throw new Error(
|
||||
`maxzoom must be an integer in [${EXTRACT_MIN_ZOOM}, ${EXTRACT_MAX_ZOOM}]`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// go-pmtiles output format isn't stable across versions — parse loosely and
|
||||
// fall back to zeros. The extract can still proceed without an estimate.
|
||||
private parseDryRunOutput(output: string): { tiles: number; bytes: number } {
|
||||
let bytes = 0
|
||||
let tiles = 0
|
||||
|
||||
const byteLine = output.match(/archive\s+size\s+of\s+([\d,.]+)\s*(B|KB|MB|GB|TB|bytes?)?/i)
|
||||
if (byteLine) {
|
||||
const raw = parseFloat(byteLine[1].replace(/,/g, ''))
|
||||
const unit = (byteLine[2] ?? 'B').toUpperCase()
|
||||
const multipliers: Record<string, number> = {
|
||||
B: 1,
|
||||
BYTE: 1,
|
||||
BYTES: 1,
|
||||
KB: 1_000,
|
||||
MB: 1_000_000,
|
||||
GB: 1_000_000_000,
|
||||
TB: 1_000_000_000_000,
|
||||
}
|
||||
bytes = Math.round(raw * (multipliers[unit] ?? 1))
|
||||
}
|
||||
|
||||
const tileLine = output.match(/(?:tiles\s+to\s+extract|tiles)[^\d]*([\d,]+)/i)
|
||||
if (tileLine) {
|
||||
tiles = parseInt(tileLine[1].replace(/,/g, ''), 10) || 0
|
||||
}
|
||||
|
||||
return { tiles, bytes }
|
||||
}
|
||||
|
||||
async delete(file: string): Promise<void> {
|
||||
let fileName = file
|
||||
if (!fileName.endsWith('.pmtiles')) {
|
||||
fileName += '.pmtiles'
|
||||
}
|
||||
|
||||
if (fileName === WORLD_BASEMAP_FILENAME) {
|
||||
throw new Error('The world basemap cannot be deleted')
|
||||
}
|
||||
|
||||
const basePath = resolve(join(this.baseDirPath, 'pmtiles'))
|
||||
const fullPath = resolve(join(basePath, fileName))
|
||||
|
||||
|
|
@ -573,3 +874,16 @@ export class MapService implements IMapService {
|
|||
return baseUrl
|
||||
}
|
||||
}
|
||||
|
||||
function findExactGroupMatch(
|
||||
countries: CountryCode[],
|
||||
groups: CountryGroup[]
|
||||
): CountryGroup | null {
|
||||
return (
|
||||
groups.find(
|
||||
(g) =>
|
||||
g.countries.length === countries.length &&
|
||||
g.countries.every((c, i) => c === countries[i])
|
||||
) ?? null
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -112,3 +112,31 @@ export const applyAllContentUpdatesValidator = vine.compile(
|
|||
.minLength(1),
|
||||
})
|
||||
)
|
||||
|
||||
// --- Map extract (regional pmtiles download) ---
|
||||
|
||||
// ISO 3166-1 alpha-2, 2 letters. Loose regex; CountriesService.resolveCodes
|
||||
// does the authoritative check against the polygon dataset.
|
||||
const countryCodeSchema = vine
|
||||
.string()
|
||||
.trim()
|
||||
.toUpperCase()
|
||||
.regex(/^[A-Z]{2}$/)
|
||||
|
||||
const countriesArraySchema = vine.array(countryCodeSchema).minLength(1).maxLength(300)
|
||||
|
||||
export const mapExtractPreflightValidator = vine.compile(
|
||||
vine.object({
|
||||
countries: countriesArraySchema.clone(),
|
||||
maxzoom: vine.number().min(0).max(15).optional(),
|
||||
})
|
||||
)
|
||||
|
||||
export const mapExtractValidator = vine.compile(
|
||||
vine.object({
|
||||
countries: countriesArraySchema.clone(),
|
||||
maxzoom: vine.number().min(0).max(15).optional(),
|
||||
label: vine.string().trim().minLength(1).maxLength(64).optional(),
|
||||
estimatedBytes: vine.number().min(0).optional(),
|
||||
})
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import type { CommandOptions } from '@adonisjs/core/types/ace'
|
|||
import { Worker } from 'bullmq'
|
||||
import queueConfig from '#config/queue'
|
||||
import { RunDownloadJob } from '#jobs/run_download_job'
|
||||
import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job'
|
||||
import { DownloadModelJob } from '#jobs/download_model_job'
|
||||
import { RunBenchmarkJob } from '#jobs/run_benchmark_job'
|
||||
import { EmbedFileJob } from '#jobs/embed_file_job'
|
||||
|
|
@ -126,6 +127,7 @@ export default class QueueWork extends BaseCommand {
|
|||
const queues = new Map<string, string>()
|
||||
|
||||
handlers.set(RunDownloadJob.key, new RunDownloadJob())
|
||||
handlers.set(RunExtractPmtilesJob.key, new RunExtractPmtilesJob())
|
||||
handlers.set(DownloadModelJob.key, new DownloadModelJob())
|
||||
handlers.set(RunBenchmarkJob.key, new RunBenchmarkJob())
|
||||
handlers.set(EmbedFileJob.key, new EmbedFileJob())
|
||||
|
|
@ -133,6 +135,7 @@ export default class QueueWork extends BaseCommand {
|
|||
handlers.set(CheckServiceUpdatesJob.key, new CheckServiceUpdatesJob())
|
||||
|
||||
queues.set(RunDownloadJob.key, RunDownloadJob.queue)
|
||||
queues.set(RunExtractPmtilesJob.key, RunExtractPmtilesJob.queue)
|
||||
queues.set(DownloadModelJob.key, DownloadModelJob.queue)
|
||||
queues.set(RunBenchmarkJob.key, RunBenchmarkJob.queue)
|
||||
queues.set(EmbedFileJob.key, EmbedFileJob.queue)
|
||||
|
|
@ -149,6 +152,9 @@ export default class QueueWork extends BaseCommand {
|
|||
private getConcurrencyForQueue(queueName: string): number {
|
||||
const concurrencyMap: Record<string, number> = {
|
||||
[RunDownloadJob.queue]: 3,
|
||||
// pmtiles extract hits the Protomaps CDN with many parallel range reads per job;
|
||||
// cap concurrency at 2 so a second extract doesn't starve the first.
|
||||
[RunExtractPmtilesJob.queue]: 2,
|
||||
[DownloadModelJob.queue]: 2, // Lower concurrency for resource-intensive model downloads
|
||||
[RunBenchmarkJob.queue]: 1, // Run benchmarks one at a time for accurate results
|
||||
[EmbedFileJob.queue]: 2, // Lower concurrency for embedding jobs, can be resource intensive
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
export const PMTILES_BINARY_PATH = '/usr/local/bin/pmtiles'
|
||||
|
||||
// Clamp these so a user can't ask for nonsense that never extracts
|
||||
export const EXTRACT_MIN_ZOOM = 0
|
||||
export const EXTRACT_MAX_ZOOM = 15
|
||||
export const EXTRACT_DEFAULT_MAX_ZOOM = 15
|
||||
|
||||
// Low-zoom global fallback extracted once during base-asset setup (~15 MB). Layered
|
||||
// underneath regional extracts so the map isn't grey outside a region's polygon.
|
||||
export const WORLD_BASEMAP_FILENAME = 'world.pmtiles'
|
||||
export const WORLD_BASEMAP_MAX_ZOOM = 5
|
||||
export const WORLD_BASEMAP_SOURCE_NAME = 'world'
|
||||
|
||||
export interface PmtilesExtractArgOptions {
|
||||
sourceUrl: string
|
||||
outputFilepath: string
|
||||
regionFilepath?: string
|
||||
maxzoom?: number
|
||||
dryRun?: boolean
|
||||
downloadThreads?: number
|
||||
overfetch?: number
|
||||
}
|
||||
|
||||
export function buildPmtilesExtractArgs(opts: PmtilesExtractArgOptions): string[] {
|
||||
const args = ['extract', opts.sourceUrl, opts.outputFilepath]
|
||||
if (opts.regionFilepath) args.push(`--region=${opts.regionFilepath}`)
|
||||
if (typeof opts.maxzoom === 'number') args.push(`--maxzoom=${opts.maxzoom}`)
|
||||
if (opts.dryRun) args.push('--dry-run')
|
||||
if (typeof opts.downloadThreads === 'number') args.push(`--download-threads=${opts.downloadThreads}`)
|
||||
if (typeof opts.overfetch === 'number') args.push(`--overfetch=${opts.overfetch}`)
|
||||
return args
|
||||
}
|
||||
|
|
@ -0,0 +1,384 @@
|
|||
import { useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
import { IconCheck, IconSearch, IconX } from '@tabler/icons-react'
|
||||
import StyledModal, { StyledModalProps } from './StyledModal'
|
||||
import LoadingSpinner from './LoadingSpinner'
|
||||
import api from '~/lib/api'
|
||||
import { formatBytes } from '~/lib/util'
|
||||
import classNames from '~/lib/classNames'
|
||||
import {
|
||||
EXTRACT_DEFAULT_MAX_ZOOM,
|
||||
EXTRACT_MAX_ZOOM,
|
||||
EXTRACT_MIN_ZOOM,
|
||||
} from '../../constants/map_regions'
|
||||
import type {
|
||||
Country,
|
||||
CountryCode,
|
||||
CountryGroup,
|
||||
MapExtractPreflight,
|
||||
} from '../../types/maps'
|
||||
|
||||
export type CountryPickerModalProps = Omit<
|
||||
StyledModalProps,
|
||||
| 'onConfirm'
|
||||
| 'open'
|
||||
| 'confirmText'
|
||||
| 'cancelText'
|
||||
| 'confirmVariant'
|
||||
| 'children'
|
||||
| 'title'
|
||||
| 'large'
|
||||
> & {
|
||||
onDownloadStart?: () => void
|
||||
}
|
||||
|
||||
const CountryPickerModal: React.FC<CountryPickerModalProps> = ({
|
||||
onDownloadStart,
|
||||
...modalProps
|
||||
}) => {
|
||||
const [selected, setSelected] = useState<Set<CountryCode>>(new Set())
|
||||
const [search, setSearch] = useState('')
|
||||
const [maxzoom, setMaxzoom] = useState<number>(EXTRACT_DEFAULT_MAX_ZOOM)
|
||||
const [preflight, setPreflight] = useState<MapExtractPreflight | null>(null)
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [downloading, setDownloading] = useState(false)
|
||||
const [errorMessage, setErrorMessage] = useState<string | null>(null)
|
||||
const preflightRequestIdRef = useRef(0)
|
||||
|
||||
const { data: countries = [], isLoading: countriesLoading } = useQuery({
|
||||
queryKey: ['maps-countries'],
|
||||
queryFn: () => api.listCountries(),
|
||||
staleTime: Infinity,
|
||||
})
|
||||
|
||||
const { data: groups = [] } = useQuery({
|
||||
queryKey: ['maps-country-groups'],
|
||||
queryFn: () => api.listCountryGroups(),
|
||||
staleTime: Infinity,
|
||||
})
|
||||
|
||||
const grouped = useMemo(() => {
|
||||
const q = search.trim().toLowerCase()
|
||||
const filtered = q
|
||||
? countries.filter(
|
||||
(c) => c.name.toLowerCase().includes(q) || c.code.toLowerCase().includes(q)
|
||||
)
|
||||
: countries
|
||||
|
||||
const buckets: Record<string, Country[]> = {}
|
||||
for (const country of filtered) {
|
||||
if (!buckets[country.continent]) buckets[country.continent] = []
|
||||
buckets[country.continent].push(country)
|
||||
}
|
||||
return Object.entries(buckets).sort(([a], [b]) => a.localeCompare(b))
|
||||
}, [countries, search])
|
||||
|
||||
const selectedCountries = useMemo(
|
||||
() => countries.filter((c) => selected.has(c.code)),
|
||||
[countries, selected]
|
||||
)
|
||||
|
||||
function toggleCountry(code: CountryCode) {
|
||||
setSelected((prev) => {
|
||||
const next = new Set(prev)
|
||||
if (next.has(code)) next.delete(code)
|
||||
else next.add(code)
|
||||
return next
|
||||
})
|
||||
}
|
||||
|
||||
function toggleGroup(group: CountryGroup) {
|
||||
setSelected((prev) => {
|
||||
const next = new Set(prev)
|
||||
const allIn = group.countries.every((c) => next.has(c))
|
||||
if (allIn) {
|
||||
group.countries.forEach((c) => next.delete(c))
|
||||
} else {
|
||||
group.countries.forEach((c) => next.add(c))
|
||||
}
|
||||
return next
|
||||
})
|
||||
}
|
||||
|
||||
function clearAll() {
|
||||
setSelected(new Set())
|
||||
}
|
||||
|
||||
// Auto-refresh the preflight whenever selection or maxzoom changes. Debounced
|
||||
// so rapid multi-select clicks collapse into a single CDN round-trip, and
|
||||
// stale-safe via requestId so an earlier slow response can't clobber a later one.
|
||||
useEffect(() => {
|
||||
if (selected.size === 0) {
|
||||
setPreflight(null)
|
||||
setErrorMessage(null)
|
||||
setLoading(false)
|
||||
preflightRequestIdRef.current++
|
||||
return
|
||||
}
|
||||
|
||||
const requestId = ++preflightRequestIdRef.current
|
||||
setLoading(true)
|
||||
setErrorMessage(null)
|
||||
const timer = setTimeout(async () => {
|
||||
try {
|
||||
const res = await api.extractMapPreflight({
|
||||
countries: [...selected],
|
||||
maxzoom,
|
||||
})
|
||||
if (requestId !== preflightRequestIdRef.current) return
|
||||
if (!res) throw new Error('Preflight returned no data')
|
||||
setPreflight(res)
|
||||
} catch (err: any) {
|
||||
if (requestId !== preflightRequestIdRef.current) return
|
||||
console.error('Preflight failed:', err)
|
||||
setErrorMessage(err?.message ?? 'Estimate failed')
|
||||
} finally {
|
||||
if (requestId === preflightRequestIdRef.current) setLoading(false)
|
||||
}
|
||||
}, 400)
|
||||
|
||||
return () => clearTimeout(timer)
|
||||
}, [selected, maxzoom])
|
||||
|
||||
async function startDownload() {
|
||||
if (selected.size === 0) {
|
||||
setErrorMessage('Pick at least one country before downloading.')
|
||||
return
|
||||
}
|
||||
if (loading || !preflight) {
|
||||
setErrorMessage('Still estimating size — hold on a moment.')
|
||||
return
|
||||
}
|
||||
try {
|
||||
setDownloading(true)
|
||||
setErrorMessage(null)
|
||||
await api.extractMapRegion({
|
||||
countries: [...selected],
|
||||
maxzoom,
|
||||
estimatedBytes: preflight?.bytes,
|
||||
})
|
||||
onDownloadStart?.()
|
||||
} catch (err: any) {
|
||||
console.error('Extract dispatch failed:', err)
|
||||
setErrorMessage(err?.message ?? 'Download failed')
|
||||
} finally {
|
||||
setDownloading(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<StyledModal
|
||||
{...modalProps}
|
||||
title="Download map by country or region"
|
||||
open={true}
|
||||
confirmText="Start Download"
|
||||
confirmIcon="IconDownload"
|
||||
cancelText="Cancel"
|
||||
confirmVariant="primary"
|
||||
confirmLoading={loading || downloading}
|
||||
cancelLoading={loading || downloading}
|
||||
onConfirm={startDownload}
|
||||
large
|
||||
>
|
||||
<div className="flex flex-col text-left gap-4 min-h-[60vh]">
|
||||
<div className="flex gap-3 items-stretch">
|
||||
<div className="relative flex-1">
|
||||
<IconSearch className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-text-muted" />
|
||||
<input
|
||||
type="text"
|
||||
value={search}
|
||||
onChange={(e) => setSearch(e.target.value)}
|
||||
placeholder={`Search ${countries.length} countries...`}
|
||||
className="w-full pl-9 pr-3 py-2 rounded-md border border-border-default bg-surface-primary text-text-primary text-sm focus:outline-none focus:ring-2 focus:ring-desert-green"
|
||||
/>
|
||||
</div>
|
||||
{selected.size > 0 && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={clearAll}
|
||||
className="text-sm text-text-muted hover:text-text-primary px-3 cursor-pointer"
|
||||
>
|
||||
Clear all
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{groups.length > 0 && (
|
||||
<div>
|
||||
<p className="text-xs uppercase tracking-wide text-text-muted mb-2">
|
||||
Quick picks
|
||||
</p>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{groups.map((group) => {
|
||||
const allIn =
|
||||
group.countries.length > 0 &&
|
||||
group.countries.every((c) => selected.has(c))
|
||||
return (
|
||||
<button
|
||||
key={group.id}
|
||||
type="button"
|
||||
onClick={() => toggleGroup(group)}
|
||||
className={classNames(
|
||||
'px-3 py-1.5 rounded-full text-xs font-medium border transition-colors cursor-pointer',
|
||||
allIn
|
||||
? 'bg-desert-green text-white border-desert-green'
|
||||
: 'bg-surface-primary text-text-primary border-border-default hover:border-desert-green'
|
||||
)}
|
||||
>
|
||||
{allIn && <IconCheck className="inline w-3 h-3 mr-1" />}
|
||||
{group.name}{' '}
|
||||
<span className="opacity-60">({group.countries.length})</span>
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex-1 overflow-y-auto max-h-96 border border-border-default rounded-md bg-surface-secondary">
|
||||
{countriesLoading ? (
|
||||
<div className="flex items-center justify-center h-40">
|
||||
<LoadingSpinner />
|
||||
</div>
|
||||
) : grouped.length === 0 ? (
|
||||
<p className="text-text-muted text-sm p-6 text-center">
|
||||
No countries match "{search}".
|
||||
</p>
|
||||
) : (
|
||||
grouped.map(([continent, list]) => (
|
||||
<div key={continent}>
|
||||
<div className="sticky top-0 bg-surface-secondary border-b border-border-default px-4 py-2 text-xs uppercase tracking-wide text-text-muted font-semibold z-10">
|
||||
{continent}
|
||||
</div>
|
||||
<ul>
|
||||
{list.map((country) => {
|
||||
const isSelected = selected.has(country.code)
|
||||
return (
|
||||
<li key={country.code}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => toggleCountry(country.code)}
|
||||
className={classNames(
|
||||
'w-full flex items-center gap-3 px-4 py-2 text-left text-sm transition-colors cursor-pointer',
|
||||
isSelected
|
||||
? 'bg-desert-green/10 hover:bg-desert-green/15'
|
||||
: 'hover:bg-surface-primary'
|
||||
)}
|
||||
>
|
||||
<span
|
||||
className={classNames(
|
||||
'w-4 h-4 rounded border flex items-center justify-center shrink-0',
|
||||
isSelected
|
||||
? 'bg-desert-green border-desert-green'
|
||||
: 'border-border-default'
|
||||
)}
|
||||
>
|
||||
{isSelected && <IconCheck className="w-3 h-3 text-white" />}
|
||||
</span>
|
||||
<span className="flex-1 text-text-primary">{country.name}</span>
|
||||
<span className="text-xs font-mono text-text-muted">
|
||||
{country.code}
|
||||
</span>
|
||||
</button>
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
</ul>
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
|
||||
{selectedCountries.length > 0 && (
|
||||
<div>
|
||||
<p className="text-xs uppercase tracking-wide text-text-muted mb-2">
|
||||
{selectedCountries.length} selected
|
||||
</p>
|
||||
<div className="flex flex-wrap gap-2 max-h-24 overflow-y-auto">
|
||||
{selectedCountries.map((country) => (
|
||||
<span
|
||||
key={country.code}
|
||||
className="inline-flex items-center gap-1 px-2 py-1 rounded bg-desert-green text-white text-xs"
|
||||
>
|
||||
{country.name}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => toggleCountry(country.code)}
|
||||
className="hover:bg-white/20 rounded cursor-pointer"
|
||||
aria-label={`Remove ${country.name}`}
|
||||
>
|
||||
<IconX className="w-3 h-3" />
|
||||
</button>
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div>
|
||||
<label className="block text-sm text-text-primary font-medium mb-2">
|
||||
Max zoom level: <span className="font-mono">{maxzoom}</span>
|
||||
</label>
|
||||
<input
|
||||
type="range"
|
||||
min={EXTRACT_MIN_ZOOM}
|
||||
max={EXTRACT_MAX_ZOOM}
|
||||
step={1}
|
||||
value={maxzoom}
|
||||
onChange={(e) => setMaxzoom(parseInt(e.target.value, 10))}
|
||||
className="w-full accent-desert-green"
|
||||
disabled={loading || downloading}
|
||||
/>
|
||||
<div className="flex justify-between text-xs text-text-muted mt-1 font-mono">
|
||||
<span>z{EXTRACT_MIN_ZOOM} (world)</span>
|
||||
<span>z{EXTRACT_MAX_ZOOM} (street)</span>
|
||||
</div>
|
||||
<p className="text-xs text-text-muted mt-2">
|
||||
Lower zoom = smaller file, less detail. Zoom 15 shows individual streets;
|
||||
zoom 10 shows city-level detail.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="bg-surface-secondary border border-border-default rounded-md p-3 min-h-14 text-sm font-mono">
|
||||
<PreflightStatus
|
||||
errorMessage={errorMessage}
|
||||
loading={loading}
|
||||
preflight={preflight}
|
||||
hasSelection={selected.size > 0}
|
||||
/>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</StyledModal>
|
||||
)
|
||||
}
|
||||
|
||||
type PreflightStatusProps = {
|
||||
errorMessage: string | null
|
||||
loading: boolean
|
||||
preflight: MapExtractPreflight | null
|
||||
hasSelection: boolean
|
||||
}
|
||||
|
||||
function PreflightStatus({ errorMessage, loading, preflight, hasSelection }: PreflightStatusProps) {
|
||||
if (errorMessage) {
|
||||
return <p className="text-desert-red">{errorMessage}</p>
|
||||
}
|
||||
if (loading) {
|
||||
return <p className="text-text-muted">Estimating size…</p>
|
||||
}
|
||||
if (preflight) {
|
||||
return (
|
||||
<p className="text-text-primary">
|
||||
{preflight.tiles.toLocaleString()} tiles, ~{formatBytes(preflight.bytes, 1)}{' '}
|
||||
<span className="text-text-muted">(source build {preflight.source.date})</span>
|
||||
</p>
|
||||
)
|
||||
}
|
||||
if (!hasSelection) {
|
||||
return <p className="text-text-muted">Pick at least one country to estimate size.</p>
|
||||
}
|
||||
return <p className="text-text-muted">Estimating size…</p>
|
||||
}
|
||||
|
||||
export default CountryPickerModal
|
||||
|
|
@ -4,6 +4,7 @@ import { ServiceSlim } from '../../types/services'
|
|||
import { FileEntry } from '../../types/files'
|
||||
import { CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus } from '../../types/system'
|
||||
import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads'
|
||||
import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps'
|
||||
import { EmbedJobWithProgress } from '../../types/rag'
|
||||
import type { CategoryWithStatus, CollectionWithStatus, ContentUpdateCheckResult, ResourceUpdateInfo } from '../../types/collections'
|
||||
import { catchInternal } from './util'
|
||||
|
|
@ -557,6 +558,46 @@ class API {
|
|||
})()
|
||||
}
|
||||
|
||||
async listCountries() {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.get<{ countries: Country[] }>('/maps/countries')
|
||||
return response.data.countries
|
||||
})()
|
||||
}
|
||||
|
||||
async listCountryGroups() {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.get<{ groups: CountryGroup[] }>('/maps/country-groups')
|
||||
return response.data.groups
|
||||
})()
|
||||
}
|
||||
|
||||
async extractMapPreflight(params: { countries: CountryCode[]; maxzoom?: number }) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.post<MapExtractPreflight>(
|
||||
'/maps/extract-preflight',
|
||||
params
|
||||
)
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async extractMapRegion(params: {
|
||||
countries: CountryCode[]
|
||||
maxzoom?: number
|
||||
label?: string
|
||||
estimatedBytes?: number
|
||||
}) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.post<{
|
||||
message: string
|
||||
filename: string
|
||||
jobId?: string
|
||||
}>('/maps/extract', params)
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async listCuratedMapCollections() {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.get<CollectionWithStatus[]>(
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
|||
import useDownloads from '~/hooks/useDownloads'
|
||||
import StyledSectionHeader from '~/components/StyledSectionHeader'
|
||||
import CuratedCollectionCard from '~/components/CuratedCollectionCard'
|
||||
import CountryPickerModal from '~/components/CountryPickerModal'
|
||||
import type { CollectionWithStatus } from '../../../types/collections'
|
||||
import ActiveDownloads from '~/components/ActiveDownloads'
|
||||
import Alert from '~/components/Alert'
|
||||
|
|
@ -221,6 +222,23 @@ export default function MapsManager(props: {
|
|||
)
|
||||
}
|
||||
|
||||
function openCountryPickerModal() {
|
||||
openModal(
|
||||
<CountryPickerModal
|
||||
onCancel={closeAllModals}
|
||||
onDownloadStart={() => {
|
||||
invalidateDownloads()
|
||||
addNotification({
|
||||
type: 'success',
|
||||
message: 'Download queued. Watch progress below.',
|
||||
})
|
||||
closeAllModals()
|
||||
}}
|
||||
/>,
|
||||
'country-picker-modal'
|
||||
)
|
||||
}
|
||||
|
||||
async function openDownloadModal() {
|
||||
openModal(
|
||||
<DownloadURLModal
|
||||
|
|
@ -309,6 +327,21 @@ export default function MapsManager(props: {
|
|||
}}
|
||||
/>
|
||||
)}
|
||||
<Alert
|
||||
title="Download by country or region"
|
||||
message="Pick the countries you actually need — from a single country to a whole continent — and we'll pull just those tiles from the global Protomaps archive. Much smaller than the full 125 GB global map."
|
||||
type="info-inverted"
|
||||
variant="bordered"
|
||||
className="mt-8"
|
||||
icon="IconMap2"
|
||||
buttonProps={{
|
||||
variant: 'primary',
|
||||
children: 'Choose Countries',
|
||||
icon: 'IconMap2',
|
||||
onClick: openCountryPickerModal,
|
||||
}}
|
||||
/>
|
||||
|
||||
<div className="mt-8 mb-6 flex items-center justify-between">
|
||||
<StyledSectionHeader title="Curated Map Regions" className="!mb-0" />
|
||||
<StyledButton
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -80,6 +80,10 @@ router
|
|||
router.post('/download-collection', [MapsController, 'downloadCollection'])
|
||||
router.get('/global-map-info', [MapsController, 'globalMapInfo'])
|
||||
router.post('/download-global-map', [MapsController, 'downloadGlobalMap'])
|
||||
router.get('/countries', [MapsController, 'listCountries'])
|
||||
router.get('/country-groups', [MapsController, 'listCountryGroups'])
|
||||
router.post('/extract-preflight', [MapsController, 'extractPreflight'])
|
||||
router.post('/extract', [MapsController, 'extractRegion'])
|
||||
router.get('/markers', [MapsController, 'listMarkers'])
|
||||
router.post('/markers', [MapsController, 'createMarker'])
|
||||
router.patch('/markers/:id', [MapsController, 'updateMarker'])
|
||||
|
|
|
|||
|
|
@ -21,3 +21,37 @@ export type MapLayer = {
|
|||
'source-layer'?: string
|
||||
[key: string]: any
|
||||
}
|
||||
|
||||
/** ISO 3166-1 alpha-2 country code (e.g. "DE", "FR", "US"). */
|
||||
export type CountryCode = string
|
||||
|
||||
export type Country = {
|
||||
code: CountryCode
|
||||
code3: string
|
||||
name: string
|
||||
continent: string
|
||||
subregion: string
|
||||
population: number
|
||||
}
|
||||
|
||||
export type CountryGroup = {
|
||||
id: string
|
||||
name: string
|
||||
description: string
|
||||
countries: CountryCode[]
|
||||
}
|
||||
|
||||
export type MapExtractRequest = {
|
||||
countries: CountryCode[]
|
||||
maxzoom?: number
|
||||
}
|
||||
|
||||
export type MapExtractPreflight = {
|
||||
tiles: number
|
||||
bytes: number
|
||||
source: {
|
||||
url: string
|
||||
date: string
|
||||
key: string
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue