From ac9640e64b3d4bebfa85b9f2b2fac655557aa0f0 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Wed, 15 Jul 2026 10:09:33 -0700 Subject: [PATCH 1/4] feat(creator-packs): backend rail for gated creator content packs Phase 1 of Creator Packs: the app-side download rail for per-creator video ZIMs served from the private R2 bucket behind the entitlement Worker. No UI yet (Phase 2). - Thread optional requestHeaders through the download util + job so the ZIM fetch can carry `Authorization: Bearer ` on both the HEAD probe and the GET stream (Worker streams the ZIM directly, gated). - Add resourceMetadata.skip_embedding and gate the RAG/EmbedFileJob branch on it so video ZIMs are never sent to the knowledge base. - New creator_packs ManifestType + creatorPacksSpecSchema (display metadata only, no url) + CollectionManifestService wiring and getCreatorPacksWithStatus() status join. - collections/creator-packs.json catalog with the two seed packs. - CreatorPackService.installPack: ensure Kiwix (auto-install if absent), resolve the stable Worker URL, dispatch the gated download with skip_embedding. Typed result codes. - CreatorPacksController + GET/POST /api/creator-packs routes. - CREATOR_PACKS_APP_KEY / CREATOR_PACKS_WORKER_BASE env (release-injected key; app degrades to a clear not_configured error when unset). Co-Authored-By: Claude Opus 4.8 (1M context) --- admin/.env.example | 7 +- .../controllers/creator_packs_controller.ts | 49 ++++++ admin/app/jobs/run_download_job.ts | 9 +- .../services/collection_manifest_service.ts | 39 ++++- admin/app/services/creator_pack_service.ts | 139 ++++++++++++++++++ admin/app/utils/downloads.ts | 10 +- admin/app/validators/curated_collections.ts | 24 +++ admin/start/env.ts | 14 ++ admin/start/routes.ts | 8 + admin/types/collections.ts | 40 ++++- admin/types/downloads.ts | 14 ++ collections/creator-packs.json | 27 ++++ 12 files changed, 372 insertions(+), 8 deletions(-) create mode 100644 admin/app/controllers/creator_packs_controller.ts create mode 100644 admin/app/services/creator_pack_service.ts create mode 100644 collections/creator-packs.json diff --git a/admin/.env.example b/admin/.env.example index 738d505..f91ebe6 100644 --- a/admin/.env.example +++ b/admin/.env.example @@ -22,4 +22,9 @@ REDIS_PORT=6379 # Storage path for NOMAD content (ZIM files, maps, etc.) # On Windows dev, use an absolute path like: C:/nomad-storage # On Linux production, use: /opt/project-nomad/storage -NOMAD_STORAGE_PATH=/opt/project-nomad/storage \ No newline at end of file +NOMAD_STORAGE_PATH=/opt/project-nomad/storage +# Creator Packs (gated content downloads). Official release builds inject +# CREATOR_PACKS_APP_KEY at build time; leave unset for a build that can't install +# packs. CREATOR_PACKS_WORKER_BASE overrides the entitlement Worker origin. +# CREATOR_PACKS_APP_KEY= +# CREATOR_PACKS_WORKER_BASE=https://nomad-packs-worker.chris-556.workers.dev \ No newline at end of file diff --git a/admin/app/controllers/creator_packs_controller.ts b/admin/app/controllers/creator_packs_controller.ts new file mode 100644 index 0000000..0d520c8 --- /dev/null +++ b/admin/app/controllers/creator_packs_controller.ts @@ -0,0 +1,49 @@ +import { CreatorPackService } from '#services/creator_pack_service' +import { DownloadService } from '#services/download_service' +import { QueueService } from '#services/queue_service' +import logger from '@adonisjs/core/services/logger' +import type { HttpContext } from '@adonisjs/core/http' + +export default class CreatorPacksController { + private creatorPackService = new CreatorPackService() + + /** + * Catalog with per-pack status, plus live progress for any in-flight pack + * downloads (ZIM download jobs — filtered client-side by collection_ref once + * the UI needs it; for now the raw zim job list is enough for progress). + */ + async index({ response }: HttpContext) { + try { + const packs = await this.creatorPackService.listPacksWithStatus() + const downloadService = new DownloadService(QueueService.getInstance()) + const downloads = await downloadService.listDownloadJobs('zim') + return { packs, downloads } + } catch (error: any) { + logger.error('[CreatorPacksController] Failed to list creator packs:', error?.message || error) + return response.status(500).send({ message: 'Failed to load creator packs' }) + } + } + + async install({ params, response }: HttpContext) { + const packId = params.id as string + const result = await this.creatorPackService.installPack(packId) + + switch (result.code) { + case 'dispatched': + return response.status(202).send({ + message: 'Pack download started', + filename: result.filename, + }) + case 'already_installed': + return { message: 'Pack is already installed' } + case 'already_downloading': + return { message: 'Pack download is already in progress' } + case 'not_found': + return response.status(404).send({ message: `Creator pack not found: ${packId}` }) + case 'not_configured': + return response.status(503).send({ + message: 'Creator Packs are not configured on this server', + }) + } + } +} diff --git a/admin/app/jobs/run_download_job.ts b/admin/app/jobs/run_download_job.ts index c9a5999..382ab52 100644 --- a/admin/app/jobs/run_download_job.ts +++ b/admin/app/jobs/run_download_job.ts @@ -67,7 +67,7 @@ export class RunDownloadJob { } async handle(job: Job) { - const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype, resourceMetadata } = + const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype, resourceMetadata, requestHeaders } = job.data as RunDownloadJobParams // Register abort controller for this job @@ -110,6 +110,7 @@ export class RunDownloadJob { timeout, allowedMimeTypes, forceNew, + requestHeaders, signal: abortController.signal, onProgress(progress) { const progressPercent = (progress.downloadedBytes / (progress.totalBytes || 1)) * 100 @@ -208,9 +209,11 @@ export class RunDownloadJob { const zimService = new ZimService(dockerService) await zimService.downloadRemoteSuccessCallback([url], true) - // Only touch the knowledge base if AI Assistant (Ollama) is installed + // Only touch the knowledge base if AI Assistant (Ollama) is installed. + // skip_embedding opts a ZIM out entirely — Creator Pack video ZIMs are + // media galleries, not text, so they must never be embedded or KB-reconciled. const ollamaUrl = await dockerService.getServiceURL('nomad_ollama') - if (ollamaUrl) { + if (ollamaUrl && !resourceMetadata?.skip_embedding) { // A content UPDATE replaces a prior file at a DIFFERENT path // (version is in the filename). A fresh install has no prior row; // a same-version re-download keeps the same path. The two cases diff --git a/admin/app/services/collection_manifest_service.ts b/admin/app/services/collection_manifest_service.ts index ac9f6e3..d977513 100644 --- a/admin/app/services/collection_manifest_service.ts +++ b/admin/app/services/collection_manifest_service.ts @@ -8,7 +8,7 @@ import InstalledResource from '#models/installed_resource' import WikipediaSelection from '#models/wikipedia_selection' import { QueueService } from './queue_service.js' import { RunDownloadJob } from '#jobs/run_download_job' -import { zimCategoriesSpecSchema, mapsSpecSchema, wikipediaSpecSchema } from '#validators/curated_collections' +import { zimCategoriesSpecSchema, mapsSpecSchema, wikipediaSpecSchema, creatorPacksSpecSchema } from '#validators/curated_collections' import { ensureDirectoryExists, listDirectoryContents, @@ -19,8 +19,10 @@ import type { ManifestType, ZimCategoriesSpec, MapsSpec, + CreatorPacksSpec, CategoryWithStatus, CollectionWithStatus, + CreatorPackWithStatus, SpecResource, SpecTier, } from '../../types/collections.js' @@ -29,12 +31,14 @@ const SPEC_URLS: Record = { zim_categories: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/kiwix-categories.json', maps: 'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/main/collections/maps.json', wikipedia: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json', + creator_packs: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/creator-packs.json', } const VALIDATORS: Record = { zim_categories: zimCategoriesSpecSchema, maps: mapsSpecSchema, wikipedia: wikipediaSpecSchema, + creator_packs: creatorPacksSpecSchema, } export class CollectionManifestService { @@ -189,6 +193,39 @@ export class CollectionManifestService { }) } + /** + * Per-pack install status for Creator Packs. Packs are single ZIMs, so this is + * a one-resource-per-pack join (simpler than the tiered category logic): + * catalog ⋈ InstalledResource (resource_type 'zim', matched on resource_id) ⋈ + * the in-flight download queue. `available_update_version` is set when an + * installed pack's version trails the catalog (a creator published a rebuild). + */ + async getCreatorPacksWithStatus(): Promise { + const spec = await this.getSpecWithFallback('creator_packs') + if (!spec) return [] + + const installedResources = await InstalledResource.query().where('resource_type', 'zim') + const installedMap = new Map(installedResources.map((r) => [r.resource_id, r])) + const inFlightIds = await this.getInFlightZimResourceIds() + + return spec.packs.map((pack) => { + const installed = installedMap.get(pack.resource_id) + if (installed) { + const hasUpdate = installed.version !== pack.version + return { + ...pack, + status: 'installed' as const, + installed_version: installed.version, + ...(hasUpdate ? { available_update_version: pack.version } : {}), + } + } + if (inFlightIds.has(pack.resource_id)) { + return { ...pack, status: 'downloading' as const } + } + return { ...pack, status: 'available' as const } + }) + } + // ---- Tier resolution ---- static resolveTierResources(tier: SpecTier, allTiers: SpecTier[]): SpecResource[] { diff --git a/admin/app/services/creator_pack_service.ts b/admin/app/services/creator_pack_service.ts new file mode 100644 index 0000000..1ca73bf --- /dev/null +++ b/admin/app/services/creator_pack_service.ts @@ -0,0 +1,139 @@ +import env from '#start/env' +import logger from '@adonisjs/core/services/logger' +import { join } from 'node:path' +import { DockerService } from '#services/docker_service' +import { CollectionManifestService } from '#services/collection_manifest_service' +import { RunDownloadJob } from '#jobs/run_download_job' +import { ZIM_STORAGE_PATH } from '../utils/fs.js' +import { SERVICE_NAMES } from '../../constants/service_names.js' +import type { CreatorPacksSpec, CreatorPackWithStatus } from '../../types/collections.js' + +// ZIMs report as octet-stream from the Worker (Content-Type is set explicitly to +// application/octet-stream in worker/src/index.js). Mirror the ZimService allowlist. +const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream'] + +// Default entitlement Worker origin. Overridable via CREATOR_PACKS_WORKER_BASE so +// we can move to a branded packs.projectnomad.us domain without a code change. +const DEFAULT_WORKER_BASE = 'https://nomad-packs-worker.chris-556.workers.dev' + +/** + * Outcome of an install request. Discriminated on `code` so the controller can + * map each case to an HTTP status without string-matching thrown messages. + * (See feedback: typed failure codes over ad-hoc Error parsing.) + */ +export type InstallPackResult = + | { code: 'dispatched'; filename: string } + | { code: 'already_installed' } + | { code: 'already_downloading' } + | { code: 'not_found' } + | { code: 'not_configured' } + +/** + * Creator Packs install rail. Diverges from the curated-collections rail in + * exactly one way: instead of a static manifest URL, the ZIM is fetched from the + * entitlement Worker with a bearer `Authorization` header. From RunDownloadJob + * onward everything is the shared rail (resumable download → InstalledResource → + * Kiwix rebuildFromDisk). Video ZIMs opt out of KB embedding via skip_embedding. + */ +export class CreatorPackService { + private dockerService: DockerService + private manifestService: CollectionManifestService + + constructor(dockerService?: DockerService, manifestService?: CollectionManifestService) { + this.dockerService = dockerService ?? new DockerService() + this.manifestService = manifestService ?? new CollectionManifestService() + } + + private get workerBase(): string { + return (env.get('CREATOR_PACKS_WORKER_BASE') || DEFAULT_WORKER_BASE).replace(/\/+$/, '') + } + + /** Stable, per-version download URL. Not signed — the gate is the auth header. */ + packUrl(id: string, version: string): string { + return `${this.workerBase}/packs/${id}_${version}.zim` + } + + async listPacksWithStatus(): Promise { + return this.manifestService.getCreatorPacksWithStatus() + } + + /** + * Install a pack by catalog id: ensure Kiwix is present (auto-install if not), + * then dispatch the gated ZIM download onto the shared rail. + */ + async installPack(packId: string): Promise { + const appKey = env.get('CREATOR_PACKS_APP_KEY') + if (!appKey) { + logger.error('[CreatorPackService] CREATOR_PACKS_APP_KEY is not set; cannot install packs') + return { code: 'not_configured' } + } + + const spec = await this.manifestService.getSpecWithFallback('creator_packs') + const pack = spec?.packs.find((p) => p.id === packId) + if (!pack) { + return { code: 'not_found' } + } + + const { default: InstalledResource } = await import('#models/installed_resource') + const existing = await InstalledResource.query() + .where('resource_type', 'zim') + .where('resource_id', pack.resource_id) + .first() + if (existing && existing.version === pack.version) { + return { code: 'already_installed' } + } + + const url = this.packUrl(pack.id, pack.version) + if (await RunDownloadJob.getActiveByUrl(url)) { + return { code: 'already_downloading' } + } + + // Kiwix is a hard prerequisite — it's what serves the pack. Auto-install it if + // absent (its own preinstall bootstraps a mini-wiki ZIM so the container can + // start). Best-effort: if the install can't be kicked off we still download the + // pack so the bytes are on disk; Kiwix hot-reloads it once installed. + await this.ensureKiwixInstalled() + + const filename = `${pack.id}_${pack.version}.zim` + const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename) + + await RunDownloadJob.dispatch({ + url, + filepath, + timeout: 30000, + allowedMimeTypes: ZIM_MIME_TYPES, + forceNew: true, + filetype: 'zim', + title: pack.name, + totalBytes: pack.size_mb ? pack.size_mb * 1024 * 1024 : undefined, + requestHeaders: { Authorization: `Bearer ${appKey}` }, + resourceMetadata: { + resource_id: pack.resource_id, + version: pack.version, + collection_ref: 'creator-packs', + skip_embedding: true, + }, + }) + + logger.info(`[CreatorPackService] Dispatched download for pack ${pack.id} (${filename})`) + return { code: 'dispatched', filename } + } + + private async ensureKiwixInstalled(): Promise { + try { + const kiwixUrl = await this.dockerService.getServiceURL(SERVICE_NAMES.KIWIX) + if (kiwixUrl) return // already installed and resolvable + + logger.info('[CreatorPackService] Kiwix not installed; auto-installing before pack download') + const result = await this.dockerService.createContainerPreflight(SERVICE_NAMES.KIWIX) + if (!result.success) { + // e.g. "already installing" — not fatal, the pack download proceeds regardless. + logger.warn(`[CreatorPackService] Kiwix preflight did not start: ${result.message}`) + } + } catch (error: any) { + logger.warn( + `[CreatorPackService] Kiwix auto-install failed (continuing with pack download): ${error?.message || error}` + ) + } + } +} diff --git a/admin/app/utils/downloads.ts b/admin/app/utils/downloads.ts index bfe189a..f998c72 100644 --- a/admin/app/utils/downloads.ts +++ b/admin/app/utils/downloads.ts @@ -24,6 +24,7 @@ export async function doResumableDownload({ onComplete, forceNew = false, allowedMimeTypes, + requestHeaders, }: DoResumableDownloadParams): Promise { const dirname = path.dirname(filepath) await ensureDirectoryExists(dirname) @@ -41,10 +42,12 @@ export async function doResumableDownload({ appendMode = true } - // Get file info with HEAD request first + // Get file info with HEAD request first. Gated sources (Creator Packs) require + // the auth header on the HEAD too, or the probe 401s before the GET is reached. const headResponse = await axios.head(url, { signal, timeout, + headers: requestHeaders, }) // Some upstream hosts (notably download.kiwix.org for .zim files) don't set a @@ -88,7 +91,10 @@ export async function doResumableDownload({ appendMode = false } - const headers: Record = {} + // Seed with any caller-supplied headers (e.g. Creator Packs' Authorization + // bearer) so every GET attempt below — including the range-restart retry — + // carries them. The Range header composes on top. + const headers: Record = { ...requestHeaders } if (supportsRangeRequests && startByte > 0) { headers.Range = `bytes=${startByte}-` } diff --git a/admin/app/validators/curated_collections.ts b/admin/app/validators/curated_collections.ts index af0d2b9..aa578f8 100644 --- a/admin/app/validators/curated_collections.ts +++ b/admin/app/validators/curated_collections.ts @@ -68,6 +68,30 @@ export const wikipediaSpecSchema = vine.object({ ).minLength(1), }) +// ---- Creator Packs spec (versioned) ---- +// +// Display metadata only — deliberately NO `url` field. The ZIM bytes are resolved +// through the entitlement Worker at install time (CreatorPackService), never from +// this public catalog. See project_content_creator_packs. +export const creatorPacksSpecSchema = vine.object({ + spec_version: vine.string(), + packs: vine.array( + vine.object({ + id: vine.string(), + name: vine.string(), + creator: vine.string(), + description: vine.string(), + version: vine.string(), + resource_id: vine.string(), + video_count: vine.number().min(0), + size_mb: vine.number().min(0), + license_id: vine.string(), + poster_url: vine.string().url().optional(), + logo_url: vine.string().url().optional(), + }) + ).minLength(1), +}) + // ---- Wikipedia validators (used by ZimService) ---- export const wikipediaOptionSchema = vine.object({ diff --git a/admin/start/env.ts b/admin/start/env.ts index 408f7b6..40a323a 100644 --- a/admin/start/env.ts +++ b/admin/start/env.ts @@ -62,4 +62,18 @@ export default await Env.create(new URL('../', import.meta.url), { |---------------------------------------------------------- */ NOMAD_API_URL: Env.schema.string.optional(), + + /* + |---------------------------------------------------------- + | Creator Packs (gated content downloads) + |---------------------------------------------------------- + | CREATOR_PACKS_APP_KEY is the shared bearer key the entitlement Worker + | requires; official release builds inject it at build time (it is NOT + | committed to source). When unset, pack installs fail with a clear + | "not configured" error and the rest of the app is unaffected. + | CREATOR_PACKS_WORKER_BASE overrides the default Worker origin (e.g. to + | point at a branded packs.projectnomad.us domain later). + */ + CREATOR_PACKS_APP_KEY: Env.schema.string.optional(), + CREATOR_PACKS_WORKER_BASE: Env.schema.string.optional(), }) diff --git a/admin/start/routes.ts b/admin/start/routes.ts index ca068cc..37a954c 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -19,6 +19,7 @@ import SettingsController from '#controllers/settings_controller' import SupplyDepotController from '#controllers/supply_depot_controller' import SystemController from '#controllers/system_controller' import CollectionUpdatesController from '#controllers/collection_updates_controller' +import CreatorPacksController from '#controllers/creator_packs_controller' import ZimController from '#controllers/zim_controller' import router from '@adonisjs/core/services/router' import transmit from '@adonisjs/transmit/services/main' @@ -225,6 +226,13 @@ router }) .prefix('/api/zim') +router + .group(() => { + router.get('/', [CreatorPacksController, 'index']) + router.post('/:id/install', [CreatorPacksController, 'install']) + }) + .prefix('/api/creator-packs') + router .group(() => { router.post('/run', [BenchmarkController, 'run']) diff --git a/admin/types/collections.ts b/admin/types/collections.ts index b8fbdb4..f969f21 100644 --- a/admin/types/collections.ts +++ b/admin/types/collections.ts @@ -58,7 +58,45 @@ export type WikipediaSpec = { options: WikipediaOption[] } -export type ManifestType = 'zim_categories' | 'maps' | 'wikipedia' +// ---- Creator Packs spec ---- +// +// A creator pack is a branded ZIM of a creator's curated YouTube videos, served +// offline through Kiwix. The public catalog carries display metadata ONLY — there +// is NO download url here. The bytes live in a private R2 bucket behind an +// entitlement Worker; CreatorPackService resolves the (stable) Worker URL and +// sends the auth header at install time. See project_content_creator_packs. +export type CreatorPack = { + id: string + name: string + creator: string + description: string + /** YYYY-MM; combines with id as `_.zim` (parseZimFilename). */ + version: string + /** InstalledResource.resource_id used for install/status reconciliation. */ + resource_id: string + video_count: number + size_mb: number + license_id: string + /** Optional card art (Phase 2). The Kiwix grid icon is baked into the ZIM. */ + poster_url?: string + logo_url?: string +} + +export type CreatorPacksSpec = { + spec_version: string + packs: CreatorPack[] +} + +export type CreatorPackStatus = 'installed' | 'downloading' | 'available' + +export type CreatorPackWithStatus = CreatorPack & { + status: CreatorPackStatus + installed_version?: string + /** Set when an installed pack has a newer catalog version available. */ + available_update_version?: string +} + +export type ManifestType = 'zim_categories' | 'maps' | 'wikipedia' | 'creator_packs' export type ResourceStatus = 'installed' | 'not_installed' | 'update_available' diff --git a/admin/types/downloads.ts b/admin/types/downloads.ts index 0bda197..669df9e 100644 --- a/admin/types/downloads.ts +++ b/admin/types/downloads.ts @@ -7,6 +7,13 @@ export type DoResumableDownloadParams = { onProgress?: (progress: DoResumableDownloadProgress) => void onComplete?: (url: string, path: string) => void | Promise forceNew?: boolean + /** + * Extra HTTP request headers sent on BOTH the HEAD probe and the GET stream. + * Used by Creator Packs to carry the `Authorization: Bearer ` the + * entitlement Worker requires; the Range header still composes on top for + * resumed downloads. Kept generic so any gated source can reuse it. + */ + requestHeaders?: Record } export type DoResumableDownloadWithRetryParams = DoResumableDownloadParams & { @@ -48,6 +55,13 @@ export type RunDownloadJobParams = Omit< * `failed` handler) so manual downloads never touch the counter. */ auto?: boolean + /** + * Skip the RAG/knowledge-base ingestion branch in RunDownloadJob.onComplete. + * Set for video ZIMs (Creator Packs) — they are media galleries, not text, + * and must never be dispatched to EmbedFileJob or the replacement-reconcile + * path. The Kiwix rebuild still runs so the pack appears in the grid. + */ + skip_embedding?: boolean } } diff --git a/collections/creator-packs.json b/collections/creator-packs.json new file mode 100644 index 0000000..d3529bf --- /dev/null +++ b/collections/creator-packs.json @@ -0,0 +1,27 @@ +{ + "spec_version": "1", + "packs": [ + { + "id": "project-nomad", + "name": "Project NOMAD", + "creator": "Project NOMAD", + "description": "Official Project NOMAD videos, packaged for offline viewing on your NOMAD.", + "version": "2026-07", + "resource_id": "project-nomad", + "video_count": 5, + "size_mb": 178, + "license_id": "nomad-creator-pack-1.0" + }, + { + "id": "crosstalk-solutions", + "name": "Crosstalk Solutions", + "creator": "Chris Sherwood", + "description": "Chris Sherwood's networking, UniFi & home lab videos, for offline NOMAD.", + "version": "2026-07", + "resource_id": "crosstalk-solutions", + "video_count": 30, + "size_mb": 2899, + "license_id": "nomad-creator-pack-1.0" + } + ] +} From 326f947b2b02f08f54c562af0280ab75d55b48dd Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Wed, 15 Jul 2026 11:03:02 -0700 Subject: [PATCH 2/4] =?UTF-8?q?feat(creator-packs):=20Phase=202=20UI=20?= =?UTF-8?q?=E2=80=94=20Content=20Explorer,=20settings=20page,=20wizard=20s?= =?UTF-8?q?tep?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the three user-facing surfaces for Creator Packs on top of the Phase 1 backend rail. All surfaces HIDE entirely on builds without the release-injected key (configured=false) so a fork never shows a broken install button. - Backend: CreatorPackService.isConfigured() + `configured` flag on the GET /api/creator-packs response; /settings/creator-packs route + controller action. - api client + useCreatorPacks hook (shared cached source of configured/packs/ downloads for every surface) + CreatorPackCard (status badges, art fallback, update pill) + CreatorPacksSection (install-on-click grid + confirm modal with the license note), reused by the Content Explorer block and settings page. - Content Explorer: Creator Packs section before Additional Content. - Settings: new /settings/creator-packs manage page + conditional nav item. - Easy Setup wizard: new Creator Packs step after Content. Replaces the fragile hardcoded AI-skip math with a computed activeSteps list so the two optional steps (Creator Packs + AI) navigate correctly in every on/off combination; selection state, storage projection, review summary, and finish-install wired. Browser-tested on NOMAD3 across all surfaces (configured + fork paths), incl. live install→download→cancel and AI-on/off wizard renumbering. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../controllers/creator_packs_controller.ts | 3 +- admin/app/controllers/settings_controller.ts | 4 + admin/app/services/creator_pack_service.ts | 10 ++ admin/inertia/components/CreatorPackCard.tsx | 96 ++++++++++ .../components/CreatorPacksSection.tsx | 100 +++++++++++ admin/inertia/hooks/useCreatorPacks.ts | 45 +++++ admin/inertia/layouts/SettingsLayout.tsx | 6 + admin/inertia/lib/api.ts | 22 ++- admin/inertia/pages/easy-setup/index.tsx | 165 ++++++++++++++---- .../inertia/pages/settings/creator-packs.tsx | 37 ++++ .../pages/settings/zim/remote-explorer.tsx | 4 + admin/start/routes.ts | 1 + 12 files changed, 458 insertions(+), 35 deletions(-) create mode 100644 admin/inertia/components/CreatorPackCard.tsx create mode 100644 admin/inertia/components/CreatorPacksSection.tsx create mode 100644 admin/inertia/hooks/useCreatorPacks.ts create mode 100644 admin/inertia/pages/settings/creator-packs.tsx diff --git a/admin/app/controllers/creator_packs_controller.ts b/admin/app/controllers/creator_packs_controller.ts index 0d520c8..0a90a10 100644 --- a/admin/app/controllers/creator_packs_controller.ts +++ b/admin/app/controllers/creator_packs_controller.ts @@ -14,10 +14,11 @@ export default class CreatorPacksController { */ async index({ response }: HttpContext) { try { + const configured = this.creatorPackService.isConfigured() const packs = await this.creatorPackService.listPacksWithStatus() const downloadService = new DownloadService(QueueService.getInstance()) const downloads = await downloadService.listDownloadJobs('zim') - return { packs, downloads } + return { configured, packs, downloads } } catch (error: any) { logger.error('[CreatorPacksController] Failed to list creator packs:', error?.message || error) return response.status(500).send({ message: 'Failed to load creator packs' }) diff --git a/admin/app/controllers/settings_controller.ts b/admin/app/controllers/settings_controller.ts index 4675d4b..7106972 100644 --- a/admin/app/controllers/settings_controller.ts +++ b/admin/app/controllers/settings_controller.ts @@ -99,6 +99,10 @@ export default class SettingsController { return inertia.render('settings/zim/remote-explorer') } + async creatorPacks({ inertia }: HttpContext) { + return inertia.render('settings/creator-packs') + } + async benchmark({ inertia }: HttpContext) { const latestResult = await this.benchmarkService.getLatestResult() const status = this.benchmarkService.getStatus() diff --git a/admin/app/services/creator_pack_service.ts b/admin/app/services/creator_pack_service.ts index 1ca73bf..8352b4a 100644 --- a/admin/app/services/creator_pack_service.ts +++ b/admin/app/services/creator_pack_service.ts @@ -48,6 +48,16 @@ export class CreatorPackService { return (env.get('CREATOR_PACKS_WORKER_BASE') || DEFAULT_WORKER_BASE).replace(/\/+$/, '') } + /** + * Whether this build can actually install packs — i.e. the release-injected + * app key is present. Forks built from source have no key; the UI uses this to + * HIDE the Creator Packs surfaces entirely rather than show install buttons + * that would 503. The public catalog still lists, but nothing is installable. + */ + isConfigured(): boolean { + return !!env.get('CREATOR_PACKS_APP_KEY') + } + /** Stable, per-version download URL. Not signed — the gate is the auth header. */ packUrl(id: string, version: string): string { return `${this.workerBase}/packs/${id}_${version}.zim` diff --git a/admin/inertia/components/CreatorPackCard.tsx b/admin/inertia/components/CreatorPackCard.tsx new file mode 100644 index 0000000..7752169 --- /dev/null +++ b/admin/inertia/components/CreatorPackCard.tsx @@ -0,0 +1,96 @@ +import { formatBytes } from '~/lib/util' +import type { CreatorPackWithStatus } from '../../types/collections' +import classNames from 'classnames' +import { + IconChevronRight, + IconCircleCheck, + IconLoader2, + IconMovie, +} from '@tabler/icons-react' + +export interface CreatorPackCardProps { + pack: CreatorPackWithStatus + /** In-session wizard selection highlight (before anything is installed). */ + selected?: boolean + onClick?: (pack: CreatorPackWithStatus) => void +} + +const CreatorPackCard: React.FC = ({ pack, selected, onClick }) => { + const isInstalled = pack.status === 'installed' + const isDownloading = pack.status === 'downloading' + const hasUpdate = !!pack.available_update_version + const sizeBytes = pack.size_mb * 1024 * 1024 + + // Installed packs are inert unless a newer version is available (click = update). + // Selected (wizard) and available packs are always clickable. + const clickable = selected || !isInstalled || hasUpdate + + const highlighted = selected || isDownloading || (isInstalled && !hasUpdate) + + return ( +
{ + if (!clickable) return + onClick?.(pack) + }} + > +
+
+
+ {pack.logo_url ? ( + + ) : ( + + )} +

{pack.name}

+
+ {selected ? ( +
+ + Selected +
+ ) : isDownloading ? ( +
+ + Downloading +
+ ) : isInstalled ? ( +
+ + Installed +
+ ) : ( + + )} +
+
+ +

by {pack.creator}

+ +

{pack.description}

+ + {hasUpdate && ( + + Update available + + )} + +
+

+ {pack.video_count} videos | {formatBytes(sizeBytes, 0)} +

+
+
+ ) +} + +export default CreatorPackCard diff --git a/admin/inertia/components/CreatorPacksSection.tsx b/admin/inertia/components/CreatorPacksSection.tsx new file mode 100644 index 0000000..9b22183 --- /dev/null +++ b/admin/inertia/components/CreatorPacksSection.tsx @@ -0,0 +1,100 @@ +import { useState } from 'react' +import { IconMovie } from '@tabler/icons-react' +import api from '~/lib/api' +import useCreatorPacks from '~/hooks/useCreatorPacks' +import useDownloads from '~/hooks/useDownloads' +import { useNotifications } from '~/context/NotificationContext' +import CreatorPackCard from '~/components/CreatorPackCard' +import StyledModal from '~/components/StyledModal' +import { formatBytes } from '~/lib/util' +import type { CreatorPackWithStatus } from '../../types/collections' + +/** + * Install-on-click grid of Creator Packs + a confirm modal. Shared by the + * Content Explorer block and the /settings/creator-packs page. Renders NOTHING + * when the build isn't configured (fork / key unset) — a fork never sees a + * broken install button. The Easy Setup wizard does NOT use this (it needs + * selection semantics, not install-on-click) and drives CreatorPackCard itself. + */ +const CreatorPacksSection: React.FC = () => { + const { configured, packs, invalidate: invalidateCreatorPacks } = useCreatorPacks() + const { invalidate: invalidateDownloads } = useDownloads({ filetype: 'zim' }) + const { addNotification } = useNotifications() + + const [packToInstall, setPackToInstall] = useState(null) + const [installing, setInstalling] = useState(false) + + if (!configured) return null + + const handleConfirm = async () => { + if (!packToInstall) return + setInstalling(true) + try { + await api.installCreatorPack(packToInstall.id) + addNotification({ message: `Started installing "${packToInstall.name}"`, type: 'success' }) + invalidateCreatorPacks() + invalidateDownloads() + setPackToInstall(null) + } catch (error) { + console.error('Error installing creator pack:', error) + addNotification({ message: 'An error occurred while starting the install.', type: 'error' }) + } finally { + setInstalling(false) + } + } + + return ( + <> +
+
+ +
+
+

Creator Packs

+

+ Branded video collections from creators, for offline viewing +

+
+
+ + {packs.length > 0 ? ( +
+ {packs.map((pack) => ( + + ))} +
+ ) : ( +

No creator packs available.

+ )} + + !installing && setPackToInstall(null)} + onCancel={() => setPackToInstall(null)} + onConfirm={handleConfirm} + confirmText={ + packToInstall?.available_update_version ? 'Update pack' : 'Install pack' + } + confirmIcon="IconDownload" + confirmLoading={installing} + icon={} + > + {packToInstall && ( +
+

+ {packToInstall.video_count} videos from {packToInstall.creator}, about{' '} + {formatBytes(packToInstall.size_mb * 1024 * 1024, 0)}. It will download in the + background and appear in Kiwix when ready. +

+

+ Licensed content — personal use, not for redistribution. +

+
+ )} + + + ) +} + +export default CreatorPacksSection diff --git a/admin/inertia/hooks/useCreatorPacks.ts b/admin/inertia/hooks/useCreatorPacks.ts new file mode 100644 index 0000000..e75d08c --- /dev/null +++ b/admin/inertia/hooks/useCreatorPacks.ts @@ -0,0 +1,45 @@ +import { useQuery, useQueryClient } from '@tanstack/react-query' +import api from '~/lib/api' + +export const CREATOR_PACKS_KEY = 'creator-packs' + +/** + * Shared, cached source of Creator Packs state for every surface (Content + * Explorer block, settings page, settings nav gate, Easy Setup wizard step). + * + * `configured` reflects whether this build carries the release-injected app key. + * When false (forks / key unset) the surfaces HIDE themselves — the catalog is + * public so packs would otherwise list with install buttons that only 503. + * + * Polls while a pack download is in flight so the card status flips promptly; + * otherwise refetches lazily. + */ +const useCreatorPacks = (enabled = true) => { + const queryClient = useQueryClient() + + const queryData = useQuery({ + queryKey: [CREATOR_PACKS_KEY], + queryFn: () => api.getCreatorPacks(), + refetchInterval: (query) => { + const anyDownloading = query.state.data?.packs?.some((p) => p.status === 'downloading') + return anyDownloading ? 2000 : false + }, + refetchOnWindowFocus: false, + staleTime: 60_000, + enabled, + }) + + const invalidate = () => { + queryClient.invalidateQueries({ queryKey: [CREATOR_PACKS_KEY] }) + } + + return { + ...queryData, + configured: queryData.data?.configured ?? false, + packs: queryData.data?.packs ?? [], + downloads: queryData.data?.downloads ?? [], + invalidate, + } +} + +export default useCreatorPacks diff --git a/admin/inertia/layouts/SettingsLayout.tsx b/admin/inertia/layouts/SettingsLayout.tsx index 6dc868d..c367003 100644 --- a/admin/inertia/layouts/SettingsLayout.tsx +++ b/admin/inertia/layouts/SettingsLayout.tsx @@ -8,6 +8,7 @@ import { IconGavel, IconHeart, IconMapRoute, + IconMovie, IconSettings, IconWand, IconZoom @@ -16,11 +17,15 @@ import { usePage } from '@inertiajs/react' import StyledSidebar from '~/components/StyledSidebar' import { getServiceLink } from '~/lib/navigation' import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus' +import useCreatorPacks from '~/hooks/useCreatorPacks' import { SERVICE_NAMES } from '../../constants/service_names' export default function SettingsLayout({ children }: { children: React.ReactNode }) { const { aiAssistantName } = usePage<{ aiAssistantName: string }>().props const aiAssistantInstallStatus = useServiceInstalledStatus(SERVICE_NAMES.OLLAMA) + // Only show the Creator Packs entry on builds that can actually install packs + // (release-injected key present) — a fork built from source has no key. + const { configured: creatorPacksConfigured } = useCreatorPacks() const navigation = [ ...(aiAssistantInstallStatus.isInstalled ? [{ name: aiAssistantName, href: '/settings/models', icon: IconWand, current: false }] : []), @@ -28,6 +33,7 @@ export default function SettingsLayout({ children }: { children: React.ReactNode { name: 'Benchmark', href: '/settings/benchmark', icon: IconChartBar, current: false }, { name: 'Content Explorer', href: '/settings/zim/remote-explorer', icon: IconZoom, current: false }, { name: 'Content Manager', href: '/settings/zim', icon: IconFolder, current: false }, + ...(creatorPacksConfigured ? [{ name: 'Creator Packs', href: '/settings/creator-packs', icon: IconMovie, current: false }] : []), { name: 'Maps Manager', href: '/settings/maps', icon: IconMapRoute, current: false }, { name: 'Service Logs & Metrics', diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 9782e9e..a068b53 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -6,7 +6,7 @@ import { AppAutoUpdateStatus, AutoUpdateStatus, CheckLatestVersionResult, Conten import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads' import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps' import { EmbedJobWithProgress, FileWarningsResult, StoredFileInfo } from '../../types/rag' -import type { CategoryWithStatus, CollectionWithStatus, ContentUpdateCheckResult, ResourceUpdateInfo } from '../../types/collections' +import type { CategoryWithStatus, CollectionWithStatus, ContentUpdateCheckResult, CreatorPackWithStatus, ResourceUpdateInfo } from '../../types/collections' import { catchInternal } from './util' import { NomadChatResponse, NomadInstalledModel, NomadOllamaModel, OllamaChatRequest } from '../../types/ollama' import BenchmarkResult from '#models/benchmark_result' @@ -697,6 +697,26 @@ class API { })() } + async getCreatorPacks() { + return catchInternal(async () => { + const response = await this.client.get<{ + configured: boolean + packs: CreatorPackWithStatus[] + downloads: DownloadJobWithProgress[] + }>('/creator-packs') + return response.data + })() + } + + async installCreatorPack(id: string) { + return catchInternal(async () => { + const response = await this.client.post<{ message: string; filename?: string }>( + `/creator-packs/${id}/install` + ) + return response.data + })() + } + async listDocs() { return catchInternal(async () => { const response = await this.client.get>('/docs/list') diff --git a/admin/inertia/pages/easy-setup/index.tsx b/admin/inertia/pages/easy-setup/index.tsx index da97cf8..6f75ea8 100644 --- a/admin/inertia/pages/easy-setup/index.tsx +++ b/admin/inertia/pages/easy-setup/index.tsx @@ -7,6 +7,7 @@ import api from '~/lib/api' import { ServiceSlim } from '../../../types/services' import CuratedCollectionCard from '~/components/CuratedCollectionCard' import CategoryCard from '~/components/CategoryCard' +import CreatorPackCard from '~/components/CreatorPackCard' import TierSelectionModal from '~/components/TierSelectionModal' import WikipediaSelector from '~/components/WikipediaSelector' import LoadingSpinner from '~/components/LoadingSpinner' @@ -15,6 +16,7 @@ import { IconCheck, IconChevronDown, IconChevronUp, IconCpu, IconBooks } from '@ import StorageProjectionBar from '~/components/StorageProjectionBar' import { useNotifications } from '~/context/NotificationContext' import useInternetStatus from '~/hooks/useInternetStatus' +import useCreatorPacks from '~/hooks/useCreatorPacks' import { useSystemInfo } from '~/hooks/useSystemInfo' import { getPrimaryDiskInfo } from '~/hooks/useDiskDisplayData' import classNames from 'classnames' @@ -106,7 +108,19 @@ const ADDITIONAL_TOOLS: Capability[] = [ }, ] -type WizardStep = 1 | 2 | 3 | 4 | 5 +// Stable step IDs. Creator Packs (4) and AI (5) are BOTH optional, so the set of +// active steps is computed at runtime (see `activeSteps`) and navigation walks +// that ordered list rather than doing hardcoded skip math. +type WizardStep = 1 | 2 | 3 | 4 | 5 | 6 + +const STEP_LABELS: Record = { + 1: 'Apps', + 2: 'Maps', + 3: 'Content', + 4: 'Creator Packs', + 5: 'AI', + 6: 'Review', +} const CURATED_MAP_COLLECTIONS_KEY = 'curated-map-collections' const CURATED_CATEGORIES_KEY = 'curated-categories' @@ -121,6 +135,7 @@ export default function EasySetupWizard(props: { const [currentStep, setCurrentStep] = useState(1) const [selectedServices, setSelectedServices] = useState([]) const [selectedMapCollections, setSelectedMapCollections] = useState([]) + const [selectedCreatorPacks, setSelectedCreatorPacks] = useState([]) const [selectedAiModels, setSelectedAiModels] = useState([]) // Auto-index policy for the AI Assistant Knowledge Base. Defaults to // 'Always' so a new user who keeps the default behavior gets the "just @@ -149,10 +164,13 @@ export default function EasySetupWizard(props: { const { isOnline } = useInternetStatus() const queryClient = useQueryClient() const { data: systemInfo } = useSystemInfo({ enabled: true }) + // Creator Packs are hidden entirely on builds without the release-injected key. + const { configured: creatorPacksConfigured, packs: creatorPacks } = useCreatorPacks() const anySelectionMade = selectedServices.length > 0 || selectedMapCollections.length > 0 || + selectedCreatorPacks.length > 0 || selectedTiers.size > 0 || selectedAiModels.length > 0 || (selectedWikipedia !== null && selectedWikipedia !== 'none') @@ -212,12 +230,29 @@ export default function EasySetupWizard(props: { [selectedServices, installedServices, remoteOllamaEnabled] ) + // Ordered list of the steps actually shown to THIS user. Creator Packs (4) and + // AI (5) are both optional; everything else is fixed. Navigation walks this + // list (see handleNext/handleBack), so a step's absence needs no skip math. + const activeSteps = useMemo(() => { + const steps: WizardStep[] = [1, 2, 3] + if (creatorPacksConfigured) steps.push(4) + if (isAiInSetup) steps.push(5) + steps.push(6) + return steps + }, [creatorPacksConfigured, isAiInSetup]) + const toggleMapCollection = (slug: string) => { setSelectedMapCollections((prev) => prev.includes(slug) ? prev.filter((s) => s !== slug) : [...prev, slug] ) } + const toggleCreatorPack = (id: string) => { + setSelectedCreatorPacks((prev) => + prev.includes(id) ? prev.filter((p) => p !== id) : [...prev, id] + ) + } + const toggleAiModel = (modelName: string) => { setSelectedAiModels((prev) => prev.includes(modelName) ? prev.filter((m) => m !== modelName) : [...prev, modelName] @@ -280,6 +315,14 @@ export default function EasySetupWizard(props: { }) } + // Add creator packs + selectedCreatorPacks.forEach((id) => { + const pack = creatorPacks.find((p) => p.id === id) + if (pack) { + totalBytes += pack.size_mb * 1024 * 1024 + } + }) + // Add AI models if (recommendedModels) { selectedAiModels.forEach((modelName) => { @@ -315,10 +358,12 @@ export default function EasySetupWizard(props: { }, [ selectedTiers, selectedMapCollections, + selectedCreatorPacks, selectedAiModels, selectedWikipedia, categories, mapCollections, + creatorPacks, recommendedModels, wikipediaState, ]) @@ -326,10 +371,9 @@ export default function EasySetupWizard(props: { // Get primary disk/filesystem info for storage projection const storageInfo = getPrimaryDiskInfo(systemInfo?.disk, systemInfo?.fsSize) - // Final step number (4 when AI is off, 5 when AI is on). Centralizing this - // here so canProceedToNextStep / handleNext / handleBack / the bottom-bar - // Next-vs-Finish switch all read the same value. - const finalStep: WizardStep = isAiInSetup ? 5 : 4 + // The review step is always the last active step. Read by canProceedToNextStep + // and the bottom-bar Next-vs-Finish switch. + const finalStep: WizardStep = activeSteps[activeSteps.length - 1] const canProceedToNextStep = () => { if (!isOnline) return false // Must be online to proceed @@ -337,18 +381,18 @@ export default function EasySetupWizard(props: { return currentStep < finalStep } + // Navigate to the next/previous ACTIVE step relative to the current one. Using + // a value comparison (not indexOf) keeps nav correct even if currentStep goes + // momentarily stale — e.g. the user disables AI while standing on the AI step, + // dropping it from activeSteps; the next click still lands on the right step. const handleNext = () => { - if (currentStep >= finalStep) return - // Skip the AI step (4) on forward nav when isAiInSetup is false. - const next = currentStep === 3 && !isAiInSetup ? 5 : currentStep + 1 - setCurrentStep(next as WizardStep) + const next = activeSteps.find((s) => s > currentStep) + if (next !== undefined) setCurrentStep(next) } const handleBack = () => { - if (currentStep <= 1) return - // Skip the AI step (4) on back nav when isAiInSetup is false. - const prev = currentStep === 5 && !isAiInSetup ? 3 : currentStep - 1 - setCurrentStep(prev as WizardStep) + const prev = [...activeSteps].reverse().find((s) => s < currentStep) + if (prev !== undefined) setCurrentStep(prev) } const handleFinish = async () => { @@ -408,6 +452,7 @@ export default function EasySetupWizard(props: { const downloadPromises = [ ...selectedMapCollections.map((slug) => api.downloadMapCollection(slug)), ...categoryTierPromises, + ...selectedCreatorPacks.map((id) => api.installCreatorPack(id)), ...selectedAiModels.map((modelName) => api.downloadModel(modelName)), ] @@ -471,24 +516,14 @@ export default function EasySetupWizard(props: { const renderStepIndicator = () => { // `step` is the stable WizardStep value (1=Apps, 2=Maps, 3=Content, - // 4=AI, 5=Review). `displayNumber` is the sequential position shown in - // the dot (always 1..N) so users see "1 2 3 4" when AI is off and - // "1 2 3 4 5" when AI is on, with no gap. - const baseSteps: Array<{ step: WizardStep; label: string }> = isAiInSetup - ? [ - { step: 1, label: 'Apps' }, - { step: 2, label: 'Maps' }, - { step: 3, label: 'Content' }, - { step: 4, label: 'AI' }, - { step: 5, label: 'Review' }, - ] - : [ - { step: 1, label: 'Apps' }, - { step: 2, label: 'Maps' }, - { step: 3, label: 'Content' }, - { step: 5, label: 'Review' }, - ] - const steps = baseSteps.map((s, idx) => ({ ...s, displayNumber: idx + 1 })) + // 4=Creator Packs, 5=AI, 6=Review). Only the ACTIVE steps are shown, and + // `displayNumber` is the sequential position in the dot (always 1..N) so + // there's no gap when Creator Packs and/or AI are absent. + const steps = activeSteps.map((step, idx) => ({ + step, + label: STEP_LABELS[step], + displayNumber: idx + 1, + })) return (