diff --git a/admin/.env.example b/admin/.env.example index 738d505..f91ebe6 100644 --- a/admin/.env.example +++ b/admin/.env.example @@ -22,4 +22,9 @@ REDIS_PORT=6379 # Storage path for NOMAD content (ZIM files, maps, etc.) # On Windows dev, use an absolute path like: C:/nomad-storage # On Linux production, use: /opt/project-nomad/storage -NOMAD_STORAGE_PATH=/opt/project-nomad/storage \ No newline at end of file +NOMAD_STORAGE_PATH=/opt/project-nomad/storage +# Creator Packs (gated content downloads). Official release builds inject +# CREATOR_PACKS_APP_KEY at build time; leave unset for a build that can't install +# packs. CREATOR_PACKS_WORKER_BASE overrides the entitlement Worker origin. +# CREATOR_PACKS_APP_KEY= +# CREATOR_PACKS_WORKER_BASE=https://nomad-packs-worker.chris-556.workers.dev \ No newline at end of file diff --git a/admin/app/controllers/creator_packs_controller.ts b/admin/app/controllers/creator_packs_controller.ts new file mode 100644 index 0000000..0d520c8 --- /dev/null +++ b/admin/app/controllers/creator_packs_controller.ts @@ -0,0 +1,49 @@ +import { CreatorPackService } from '#services/creator_pack_service' +import { DownloadService } from '#services/download_service' +import { QueueService } from '#services/queue_service' +import logger from '@adonisjs/core/services/logger' +import type { HttpContext } from '@adonisjs/core/http' + +export default class CreatorPacksController { + private creatorPackService = new CreatorPackService() + + /** + * Catalog with per-pack status, plus live progress for any in-flight pack + * downloads (ZIM download jobs — filtered client-side by collection_ref once + * the UI needs it; for now the raw zim job list is enough for progress). + */ + async index({ response }: HttpContext) { + try { + const packs = await this.creatorPackService.listPacksWithStatus() + const downloadService = new DownloadService(QueueService.getInstance()) + const downloads = await downloadService.listDownloadJobs('zim') + return { packs, downloads } + } catch (error: any) { + logger.error('[CreatorPacksController] Failed to list creator packs:', error?.message || error) + return response.status(500).send({ message: 'Failed to load creator packs' }) + } + } + + async install({ params, response }: HttpContext) { + const packId = params.id as string + const result = await this.creatorPackService.installPack(packId) + + switch (result.code) { + case 'dispatched': + return response.status(202).send({ + message: 'Pack download started', + filename: result.filename, + }) + case 'already_installed': + return { message: 'Pack is already installed' } + case 'already_downloading': + return { message: 'Pack download is already in progress' } + case 'not_found': + return response.status(404).send({ message: `Creator pack not found: ${packId}` }) + case 'not_configured': + return response.status(503).send({ + message: 'Creator Packs are not configured on this server', + }) + } + } +} diff --git a/admin/app/jobs/run_download_job.ts b/admin/app/jobs/run_download_job.ts index c9a5999..382ab52 100644 --- a/admin/app/jobs/run_download_job.ts +++ b/admin/app/jobs/run_download_job.ts @@ -67,7 +67,7 @@ export class RunDownloadJob { } async handle(job: Job) { - const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype, resourceMetadata } = + const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype, resourceMetadata, requestHeaders } = job.data as RunDownloadJobParams // Register abort controller for this job @@ -110,6 +110,7 @@ export class RunDownloadJob { timeout, allowedMimeTypes, forceNew, + requestHeaders, signal: abortController.signal, onProgress(progress) { const progressPercent = (progress.downloadedBytes / (progress.totalBytes || 1)) * 100 @@ -208,9 +209,11 @@ export class RunDownloadJob { const zimService = new ZimService(dockerService) await zimService.downloadRemoteSuccessCallback([url], true) - // Only touch the knowledge base if AI Assistant (Ollama) is installed + // Only touch the knowledge base if AI Assistant (Ollama) is installed. + // skip_embedding opts a ZIM out entirely — Creator Pack video ZIMs are + // media galleries, not text, so they must never be embedded or KB-reconciled. const ollamaUrl = await dockerService.getServiceURL('nomad_ollama') - if (ollamaUrl) { + if (ollamaUrl && !resourceMetadata?.skip_embedding) { // A content UPDATE replaces a prior file at a DIFFERENT path // (version is in the filename). A fresh install has no prior row; // a same-version re-download keeps the same path. The two cases diff --git a/admin/app/services/collection_manifest_service.ts b/admin/app/services/collection_manifest_service.ts index ac9f6e3..d977513 100644 --- a/admin/app/services/collection_manifest_service.ts +++ b/admin/app/services/collection_manifest_service.ts @@ -8,7 +8,7 @@ import InstalledResource from '#models/installed_resource' import WikipediaSelection from '#models/wikipedia_selection' import { QueueService } from './queue_service.js' import { RunDownloadJob } from '#jobs/run_download_job' -import { zimCategoriesSpecSchema, mapsSpecSchema, wikipediaSpecSchema } from '#validators/curated_collections' +import { zimCategoriesSpecSchema, mapsSpecSchema, wikipediaSpecSchema, creatorPacksSpecSchema } from '#validators/curated_collections' import { ensureDirectoryExists, listDirectoryContents, @@ -19,8 +19,10 @@ import type { ManifestType, ZimCategoriesSpec, MapsSpec, + CreatorPacksSpec, CategoryWithStatus, CollectionWithStatus, + CreatorPackWithStatus, SpecResource, SpecTier, } from '../../types/collections.js' @@ -29,12 +31,14 @@ const SPEC_URLS: Record = { zim_categories: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/kiwix-categories.json', maps: 'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/main/collections/maps.json', wikipedia: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json', + creator_packs: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/creator-packs.json', } const VALIDATORS: Record = { zim_categories: zimCategoriesSpecSchema, maps: mapsSpecSchema, wikipedia: wikipediaSpecSchema, + creator_packs: creatorPacksSpecSchema, } export class CollectionManifestService { @@ -189,6 +193,39 @@ export class CollectionManifestService { }) } + /** + * Per-pack install status for Creator Packs. Packs are single ZIMs, so this is + * a one-resource-per-pack join (simpler than the tiered category logic): + * catalog ⋈ InstalledResource (resource_type 'zim', matched on resource_id) ⋈ + * the in-flight download queue. `available_update_version` is set when an + * installed pack's version trails the catalog (a creator published a rebuild). + */ + async getCreatorPacksWithStatus(): Promise { + const spec = await this.getSpecWithFallback('creator_packs') + if (!spec) return [] + + const installedResources = await InstalledResource.query().where('resource_type', 'zim') + const installedMap = new Map(installedResources.map((r) => [r.resource_id, r])) + const inFlightIds = await this.getInFlightZimResourceIds() + + return spec.packs.map((pack) => { + const installed = installedMap.get(pack.resource_id) + if (installed) { + const hasUpdate = installed.version !== pack.version + return { + ...pack, + status: 'installed' as const, + installed_version: installed.version, + ...(hasUpdate ? { available_update_version: pack.version } : {}), + } + } + if (inFlightIds.has(pack.resource_id)) { + return { ...pack, status: 'downloading' as const } + } + return { ...pack, status: 'available' as const } + }) + } + // ---- Tier resolution ---- static resolveTierResources(tier: SpecTier, allTiers: SpecTier[]): SpecResource[] { diff --git a/admin/app/services/creator_pack_service.ts b/admin/app/services/creator_pack_service.ts new file mode 100644 index 0000000..1ca73bf --- /dev/null +++ b/admin/app/services/creator_pack_service.ts @@ -0,0 +1,139 @@ +import env from '#start/env' +import logger from '@adonisjs/core/services/logger' +import { join } from 'node:path' +import { DockerService } from '#services/docker_service' +import { CollectionManifestService } from '#services/collection_manifest_service' +import { RunDownloadJob } from '#jobs/run_download_job' +import { ZIM_STORAGE_PATH } from '../utils/fs.js' +import { SERVICE_NAMES } from '../../constants/service_names.js' +import type { CreatorPacksSpec, CreatorPackWithStatus } from '../../types/collections.js' + +// ZIMs report as octet-stream from the Worker (Content-Type is set explicitly to +// application/octet-stream in worker/src/index.js). Mirror the ZimService allowlist. +const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream'] + +// Default entitlement Worker origin. Overridable via CREATOR_PACKS_WORKER_BASE so +// we can move to a branded packs.projectnomad.us domain without a code change. +const DEFAULT_WORKER_BASE = 'https://nomad-packs-worker.chris-556.workers.dev' + +/** + * Outcome of an install request. Discriminated on `code` so the controller can + * map each case to an HTTP status without string-matching thrown messages. + * (See feedback: typed failure codes over ad-hoc Error parsing.) + */ +export type InstallPackResult = + | { code: 'dispatched'; filename: string } + | { code: 'already_installed' } + | { code: 'already_downloading' } + | { code: 'not_found' } + | { code: 'not_configured' } + +/** + * Creator Packs install rail. Diverges from the curated-collections rail in + * exactly one way: instead of a static manifest URL, the ZIM is fetched from the + * entitlement Worker with a bearer `Authorization` header. From RunDownloadJob + * onward everything is the shared rail (resumable download → InstalledResource → + * Kiwix rebuildFromDisk). Video ZIMs opt out of KB embedding via skip_embedding. + */ +export class CreatorPackService { + private dockerService: DockerService + private manifestService: CollectionManifestService + + constructor(dockerService?: DockerService, manifestService?: CollectionManifestService) { + this.dockerService = dockerService ?? new DockerService() + this.manifestService = manifestService ?? new CollectionManifestService() + } + + private get workerBase(): string { + return (env.get('CREATOR_PACKS_WORKER_BASE') || DEFAULT_WORKER_BASE).replace(/\/+$/, '') + } + + /** Stable, per-version download URL. Not signed — the gate is the auth header. */ + packUrl(id: string, version: string): string { + return `${this.workerBase}/packs/${id}_${version}.zim` + } + + async listPacksWithStatus(): Promise { + return this.manifestService.getCreatorPacksWithStatus() + } + + /** + * Install a pack by catalog id: ensure Kiwix is present (auto-install if not), + * then dispatch the gated ZIM download onto the shared rail. + */ + async installPack(packId: string): Promise { + const appKey = env.get('CREATOR_PACKS_APP_KEY') + if (!appKey) { + logger.error('[CreatorPackService] CREATOR_PACKS_APP_KEY is not set; cannot install packs') + return { code: 'not_configured' } + } + + const spec = await this.manifestService.getSpecWithFallback('creator_packs') + const pack = spec?.packs.find((p) => p.id === packId) + if (!pack) { + return { code: 'not_found' } + } + + const { default: InstalledResource } = await import('#models/installed_resource') + const existing = await InstalledResource.query() + .where('resource_type', 'zim') + .where('resource_id', pack.resource_id) + .first() + if (existing && existing.version === pack.version) { + return { code: 'already_installed' } + } + + const url = this.packUrl(pack.id, pack.version) + if (await RunDownloadJob.getActiveByUrl(url)) { + return { code: 'already_downloading' } + } + + // Kiwix is a hard prerequisite — it's what serves the pack. Auto-install it if + // absent (its own preinstall bootstraps a mini-wiki ZIM so the container can + // start). Best-effort: if the install can't be kicked off we still download the + // pack so the bytes are on disk; Kiwix hot-reloads it once installed. + await this.ensureKiwixInstalled() + + const filename = `${pack.id}_${pack.version}.zim` + const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename) + + await RunDownloadJob.dispatch({ + url, + filepath, + timeout: 30000, + allowedMimeTypes: ZIM_MIME_TYPES, + forceNew: true, + filetype: 'zim', + title: pack.name, + totalBytes: pack.size_mb ? pack.size_mb * 1024 * 1024 : undefined, + requestHeaders: { Authorization: `Bearer ${appKey}` }, + resourceMetadata: { + resource_id: pack.resource_id, + version: pack.version, + collection_ref: 'creator-packs', + skip_embedding: true, + }, + }) + + logger.info(`[CreatorPackService] Dispatched download for pack ${pack.id} (${filename})`) + return { code: 'dispatched', filename } + } + + private async ensureKiwixInstalled(): Promise { + try { + const kiwixUrl = await this.dockerService.getServiceURL(SERVICE_NAMES.KIWIX) + if (kiwixUrl) return // already installed and resolvable + + logger.info('[CreatorPackService] Kiwix not installed; auto-installing before pack download') + const result = await this.dockerService.createContainerPreflight(SERVICE_NAMES.KIWIX) + if (!result.success) { + // e.g. "already installing" — not fatal, the pack download proceeds regardless. + logger.warn(`[CreatorPackService] Kiwix preflight did not start: ${result.message}`) + } + } catch (error: any) { + logger.warn( + `[CreatorPackService] Kiwix auto-install failed (continuing with pack download): ${error?.message || error}` + ) + } + } +} diff --git a/admin/app/utils/downloads.ts b/admin/app/utils/downloads.ts index bfe189a..f998c72 100644 --- a/admin/app/utils/downloads.ts +++ b/admin/app/utils/downloads.ts @@ -24,6 +24,7 @@ export async function doResumableDownload({ onComplete, forceNew = false, allowedMimeTypes, + requestHeaders, }: DoResumableDownloadParams): Promise { const dirname = path.dirname(filepath) await ensureDirectoryExists(dirname) @@ -41,10 +42,12 @@ export async function doResumableDownload({ appendMode = true } - // Get file info with HEAD request first + // Get file info with HEAD request first. Gated sources (Creator Packs) require + // the auth header on the HEAD too, or the probe 401s before the GET is reached. const headResponse = await axios.head(url, { signal, timeout, + headers: requestHeaders, }) // Some upstream hosts (notably download.kiwix.org for .zim files) don't set a @@ -88,7 +91,10 @@ export async function doResumableDownload({ appendMode = false } - const headers: Record = {} + // Seed with any caller-supplied headers (e.g. Creator Packs' Authorization + // bearer) so every GET attempt below — including the range-restart retry — + // carries them. The Range header composes on top. + const headers: Record = { ...requestHeaders } if (supportsRangeRequests && startByte > 0) { headers.Range = `bytes=${startByte}-` } diff --git a/admin/app/validators/curated_collections.ts b/admin/app/validators/curated_collections.ts index af0d2b9..aa578f8 100644 --- a/admin/app/validators/curated_collections.ts +++ b/admin/app/validators/curated_collections.ts @@ -68,6 +68,30 @@ export const wikipediaSpecSchema = vine.object({ ).minLength(1), }) +// ---- Creator Packs spec (versioned) ---- +// +// Display metadata only — deliberately NO `url` field. The ZIM bytes are resolved +// through the entitlement Worker at install time (CreatorPackService), never from +// this public catalog. See project_content_creator_packs. +export const creatorPacksSpecSchema = vine.object({ + spec_version: vine.string(), + packs: vine.array( + vine.object({ + id: vine.string(), + name: vine.string(), + creator: vine.string(), + description: vine.string(), + version: vine.string(), + resource_id: vine.string(), + video_count: vine.number().min(0), + size_mb: vine.number().min(0), + license_id: vine.string(), + poster_url: vine.string().url().optional(), + logo_url: vine.string().url().optional(), + }) + ).minLength(1), +}) + // ---- Wikipedia validators (used by ZimService) ---- export const wikipediaOptionSchema = vine.object({ diff --git a/admin/start/env.ts b/admin/start/env.ts index 408f7b6..40a323a 100644 --- a/admin/start/env.ts +++ b/admin/start/env.ts @@ -62,4 +62,18 @@ export default await Env.create(new URL('../', import.meta.url), { |---------------------------------------------------------- */ NOMAD_API_URL: Env.schema.string.optional(), + + /* + |---------------------------------------------------------- + | Creator Packs (gated content downloads) + |---------------------------------------------------------- + | CREATOR_PACKS_APP_KEY is the shared bearer key the entitlement Worker + | requires; official release builds inject it at build time (it is NOT + | committed to source). When unset, pack installs fail with a clear + | "not configured" error and the rest of the app is unaffected. + | CREATOR_PACKS_WORKER_BASE overrides the default Worker origin (e.g. to + | point at a branded packs.projectnomad.us domain later). + */ + CREATOR_PACKS_APP_KEY: Env.schema.string.optional(), + CREATOR_PACKS_WORKER_BASE: Env.schema.string.optional(), }) diff --git a/admin/start/routes.ts b/admin/start/routes.ts index ca068cc..37a954c 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -19,6 +19,7 @@ import SettingsController from '#controllers/settings_controller' import SupplyDepotController from '#controllers/supply_depot_controller' import SystemController from '#controllers/system_controller' import CollectionUpdatesController from '#controllers/collection_updates_controller' +import CreatorPacksController from '#controllers/creator_packs_controller' import ZimController from '#controllers/zim_controller' import router from '@adonisjs/core/services/router' import transmit from '@adonisjs/transmit/services/main' @@ -225,6 +226,13 @@ router }) .prefix('/api/zim') +router + .group(() => { + router.get('/', [CreatorPacksController, 'index']) + router.post('/:id/install', [CreatorPacksController, 'install']) + }) + .prefix('/api/creator-packs') + router .group(() => { router.post('/run', [BenchmarkController, 'run']) diff --git a/admin/types/collections.ts b/admin/types/collections.ts index b8fbdb4..f969f21 100644 --- a/admin/types/collections.ts +++ b/admin/types/collections.ts @@ -58,7 +58,45 @@ export type WikipediaSpec = { options: WikipediaOption[] } -export type ManifestType = 'zim_categories' | 'maps' | 'wikipedia' +// ---- Creator Packs spec ---- +// +// A creator pack is a branded ZIM of a creator's curated YouTube videos, served +// offline through Kiwix. The public catalog carries display metadata ONLY — there +// is NO download url here. The bytes live in a private R2 bucket behind an +// entitlement Worker; CreatorPackService resolves the (stable) Worker URL and +// sends the auth header at install time. See project_content_creator_packs. +export type CreatorPack = { + id: string + name: string + creator: string + description: string + /** YYYY-MM; combines with id as `_.zim` (parseZimFilename). */ + version: string + /** InstalledResource.resource_id used for install/status reconciliation. */ + resource_id: string + video_count: number + size_mb: number + license_id: string + /** Optional card art (Phase 2). The Kiwix grid icon is baked into the ZIM. */ + poster_url?: string + logo_url?: string +} + +export type CreatorPacksSpec = { + spec_version: string + packs: CreatorPack[] +} + +export type CreatorPackStatus = 'installed' | 'downloading' | 'available' + +export type CreatorPackWithStatus = CreatorPack & { + status: CreatorPackStatus + installed_version?: string + /** Set when an installed pack has a newer catalog version available. */ + available_update_version?: string +} + +export type ManifestType = 'zim_categories' | 'maps' | 'wikipedia' | 'creator_packs' export type ResourceStatus = 'installed' | 'not_installed' | 'update_available' diff --git a/admin/types/downloads.ts b/admin/types/downloads.ts index 0bda197..669df9e 100644 --- a/admin/types/downloads.ts +++ b/admin/types/downloads.ts @@ -7,6 +7,13 @@ export type DoResumableDownloadParams = { onProgress?: (progress: DoResumableDownloadProgress) => void onComplete?: (url: string, path: string) => void | Promise forceNew?: boolean + /** + * Extra HTTP request headers sent on BOTH the HEAD probe and the GET stream. + * Used by Creator Packs to carry the `Authorization: Bearer ` the + * entitlement Worker requires; the Range header still composes on top for + * resumed downloads. Kept generic so any gated source can reuse it. + */ + requestHeaders?: Record } export type DoResumableDownloadWithRetryParams = DoResumableDownloadParams & { @@ -48,6 +55,13 @@ export type RunDownloadJobParams = Omit< * `failed` handler) so manual downloads never touch the counter. */ auto?: boolean + /** + * Skip the RAG/knowledge-base ingestion branch in RunDownloadJob.onComplete. + * Set for video ZIMs (Creator Packs) — they are media galleries, not text, + * and must never be dispatched to EmbedFileJob or the replacement-reconcile + * path. The Kiwix rebuild still runs so the pack appears in the grid. + */ + skip_embedding?: boolean } } diff --git a/collections/creator-packs.json b/collections/creator-packs.json new file mode 100644 index 0000000..d3529bf --- /dev/null +++ b/collections/creator-packs.json @@ -0,0 +1,27 @@ +{ + "spec_version": "1", + "packs": [ + { + "id": "project-nomad", + "name": "Project NOMAD", + "creator": "Project NOMAD", + "description": "Official Project NOMAD videos, packaged for offline viewing on your NOMAD.", + "version": "2026-07", + "resource_id": "project-nomad", + "video_count": 5, + "size_mb": 178, + "license_id": "nomad-creator-pack-1.0" + }, + { + "id": "crosstalk-solutions", + "name": "Crosstalk Solutions", + "creator": "Chris Sherwood", + "description": "Chris Sherwood's networking, UniFi & home lab videos, for offline NOMAD.", + "version": "2026-07", + "resource_id": "crosstalk-solutions", + "video_count": 30, + "size_mb": 2899, + "license_id": "nomad-creator-pack-1.0" + } + ] +}