feat(creator-packs): backend rail for gated creator content packs

Phase 1 of Creator Packs: the app-side download rail for per-creator
video ZIMs served from the private R2 bucket behind the entitlement
Worker. No UI yet (Phase 2).

- Thread optional requestHeaders through the download util + job so the
  ZIM fetch can carry `Authorization: Bearer <APP_KEY>` on both the HEAD
  probe and the GET stream (Worker streams the ZIM directly, gated).
- Add resourceMetadata.skip_embedding and gate the RAG/EmbedFileJob
  branch on it so video ZIMs are never sent to the knowledge base.
- New creator_packs ManifestType + creatorPacksSpecSchema (display
  metadata only, no url) + CollectionManifestService wiring and
  getCreatorPacksWithStatus() status join.
- collections/creator-packs.json catalog with the two seed packs.
- CreatorPackService.installPack: ensure Kiwix (auto-install if absent),
  resolve the stable Worker URL, dispatch the gated download with
  skip_embedding. Typed result codes.
- CreatorPacksController + GET/POST /api/creator-packs routes.
- CREATOR_PACKS_APP_KEY / CREATOR_PACKS_WORKER_BASE env (release-injected
  key; app degrades to a clear not_configured error when unset).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

admin/.env.example

@@ -22,4 +22,9 @@ REDIS_PORT=6379
 # Storage path for NOMAD content (ZIM files, maps, etc.)
 # On Windows dev, use an absolute path like: C:/nomad-storage
 # On Linux production, use: /opt/project-nomad/storage
-NOMAD_STORAGE_PATH=/opt/project-nomad/storage
+NOMAD_STORAGE_PATH=/opt/project-nomad/storage
+# Creator Packs (gated content downloads). Official release builds inject
+# CREATOR_PACKS_APP_KEY at build time; leave unset for a build that can't install
+# packs. CREATOR_PACKS_WORKER_BASE overrides the entitlement Worker origin.
+# CREATOR_PACKS_APP_KEY=
+# CREATOR_PACKS_WORKER_BASE=https://nomad-packs-worker.chris-556.workers.dev

admin/app/controllers/creator_packs_controller.ts

@@ -0,0 +1,49 @@
+import { CreatorPackService } from '#services/creator_pack_service'
+import { DownloadService } from '#services/download_service'
+import { QueueService } from '#services/queue_service'
+import logger from '@adonisjs/core/services/logger'
+import type { HttpContext } from '@adonisjs/core/http'
+
+export default class CreatorPacksController {
+  private creatorPackService = new CreatorPackService()
+
+  /**
+   * Catalog with per-pack status, plus live progress for any in-flight pack
+   * downloads (ZIM download jobs — filtered client-side by collection_ref once
+   * the UI needs it; for now the raw zim job list is enough for progress).
+   */
+  async index({ response }: HttpContext) {
+    try {
+      const packs = await this.creatorPackService.listPacksWithStatus()
+      const downloadService = new DownloadService(QueueService.getInstance())
+      const downloads = await downloadService.listDownloadJobs('zim')
+      return { packs, downloads }
+    } catch (error: any) {
+      logger.error('[CreatorPacksController] Failed to list creator packs:', error?.message || error)
+      return response.status(500).send({ message: 'Failed to load creator packs' })
+    }
+  }
+
+  async install({ params, response }: HttpContext) {
+    const packId = params.id as string
+    const result = await this.creatorPackService.installPack(packId)
+
+    switch (result.code) {
+      case 'dispatched':
+        return response.status(202).send({
+          message: 'Pack download started',
+          filename: result.filename,
+        })
+      case 'already_installed':
+        return { message: 'Pack is already installed' }
+      case 'already_downloading':
+        return { message: 'Pack download is already in progress' }
+      case 'not_found':
+        return response.status(404).send({ message: `Creator pack not found: ${packId}` })
+      case 'not_configured':
+        return response.status(503).send({
+          message: 'Creator Packs are not configured on this server',
+        })
+    }
+  }
+}

admin/app/jobs/run_download_job.ts

@@ -67,7 +67,7 @@ export class RunDownloadJob {
   }
 
   async handle(job: Job) {
-    const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype, resourceMetadata } =
+    const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype, resourceMetadata, requestHeaders } =
       job.data as RunDownloadJobParams
 
     // Register abort controller for this job
@@ -110,6 +110,7 @@ export class RunDownloadJob {
         timeout,
         allowedMimeTypes,
         forceNew,
+        requestHeaders,
         signal: abortController.signal,
         onProgress(progress) {
           const progressPercent = (progress.downloadedBytes / (progress.totalBytes || 1)) * 100
@@ -208,9 +209,11 @@ export class RunDownloadJob {
               const zimService = new ZimService(dockerService)
               await zimService.downloadRemoteSuccessCallback([url], true)
 
-              // Only touch the knowledge base if AI Assistant (Ollama) is installed
+              // Only touch the knowledge base if AI Assistant (Ollama) is installed.
+              // skip_embedding opts a ZIM out entirely — Creator Pack video ZIMs are
+              // media galleries, not text, so they must never be embedded or KB-reconciled.
               const ollamaUrl = await dockerService.getServiceURL('nomad_ollama')
-              if (ollamaUrl) {
+              if (ollamaUrl && !resourceMetadata?.skip_embedding) {
                 // A content UPDATE replaces a prior file at a DIFFERENT path
                 // (version is in the filename). A fresh install has no prior row;
                 // a same-version re-download keeps the same path. The two cases

admin/app/services/collection_manifest_service.ts

@@ -8,7 +8,7 @@ import InstalledResource from '#models/installed_resource'
 import WikipediaSelection from '#models/wikipedia_selection'
 import { QueueService } from './queue_service.js'
 import { RunDownloadJob } from '#jobs/run_download_job'
-import { zimCategoriesSpecSchema, mapsSpecSchema, wikipediaSpecSchema } from '#validators/curated_collections'
+import { zimCategoriesSpecSchema, mapsSpecSchema, wikipediaSpecSchema, creatorPacksSpecSchema } from '#validators/curated_collections'
 import {
   ensureDirectoryExists,
   listDirectoryContents,
@@ -19,8 +19,10 @@ import type {
   ManifestType,
   ZimCategoriesSpec,
   MapsSpec,
+  CreatorPacksSpec,
   CategoryWithStatus,
   CollectionWithStatus,
+  CreatorPackWithStatus,
   SpecResource,
   SpecTier,
 } from '../../types/collections.js'
@@ -29,12 +31,14 @@ const SPEC_URLS: Record<ManifestType, string> = {
   zim_categories: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/kiwix-categories.json',
   maps: 'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/main/collections/maps.json',
   wikipedia: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json',
+  creator_packs: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/creator-packs.json',
 }
 
 const VALIDATORS: Record<ManifestType, any> = {
   zim_categories: zimCategoriesSpecSchema,
   maps: mapsSpecSchema,
   wikipedia: wikipediaSpecSchema,
+  creator_packs: creatorPacksSpecSchema,
 }
 
 export class CollectionManifestService {
@@ -189,6 +193,39 @@ export class CollectionManifestService {
     })
   }
 
+  /**
+   * Per-pack install status for Creator Packs. Packs are single ZIMs, so this is
+   * a one-resource-per-pack join (simpler than the tiered category logic):
+   * catalog ⋈ InstalledResource (resource_type 'zim', matched on resource_id) ⋈
+   * the in-flight download queue. `available_update_version` is set when an
+   * installed pack's version trails the catalog (a creator published a rebuild).
+   */
+  async getCreatorPacksWithStatus(): Promise<CreatorPackWithStatus[]> {
+    const spec = await this.getSpecWithFallback<CreatorPacksSpec>('creator_packs')
+    if (!spec) return []
+
+    const installedResources = await InstalledResource.query().where('resource_type', 'zim')
+    const installedMap = new Map(installedResources.map((r) => [r.resource_id, r]))
+    const inFlightIds = await this.getInFlightZimResourceIds()
+
+    return spec.packs.map((pack) => {
+      const installed = installedMap.get(pack.resource_id)
+      if (installed) {
+        const hasUpdate = installed.version !== pack.version
+        return {
+          ...pack,
+          status: 'installed' as const,
+          installed_version: installed.version,
+          ...(hasUpdate ? { available_update_version: pack.version } : {}),
+        }
+      }
+      if (inFlightIds.has(pack.resource_id)) {
+        return { ...pack, status: 'downloading' as const }
+      }
+      return { ...pack, status: 'available' as const }
+    })
+  }
+
   // ---- Tier resolution ----
 
   static resolveTierResources(tier: SpecTier, allTiers: SpecTier[]): SpecResource[] {

admin/app/services/creator_pack_service.ts

@@ -0,0 +1,139 @@
+import env from '#start/env'
+import logger from '@adonisjs/core/services/logger'
+import { join } from 'node:path'
+import { DockerService } from '#services/docker_service'
+import { CollectionManifestService } from '#services/collection_manifest_service'
+import { RunDownloadJob } from '#jobs/run_download_job'
+import { ZIM_STORAGE_PATH } from '../utils/fs.js'
+import { SERVICE_NAMES } from '../../constants/service_names.js'
+import type { CreatorPacksSpec, CreatorPackWithStatus } from '../../types/collections.js'
+
+// ZIMs report as octet-stream from the Worker (Content-Type is set explicitly to
+// application/octet-stream in worker/src/index.js). Mirror the ZimService allowlist.
+const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
+
+// Default entitlement Worker origin. Overridable via CREATOR_PACKS_WORKER_BASE so
+// we can move to a branded packs.projectnomad.us domain without a code change.
+const DEFAULT_WORKER_BASE = 'https://nomad-packs-worker.chris-556.workers.dev'
+
+/**
+ * Outcome of an install request. Discriminated on `code` so the controller can
+ * map each case to an HTTP status without string-matching thrown messages.
+ * (See feedback: typed failure codes over ad-hoc Error parsing.)
+ */
+export type InstallPackResult =
+  | { code: 'dispatched'; filename: string }
+  | { code: 'already_installed' }
+  | { code: 'already_downloading' }
+  | { code: 'not_found' }
+  | { code: 'not_configured' }
+
+/**
+ * Creator Packs install rail. Diverges from the curated-collections rail in
+ * exactly one way: instead of a static manifest URL, the ZIM is fetched from the
+ * entitlement Worker with a bearer `Authorization` header. From RunDownloadJob
+ * onward everything is the shared rail (resumable download → InstalledResource →
+ * Kiwix rebuildFromDisk). Video ZIMs opt out of KB embedding via skip_embedding.
+ */
+export class CreatorPackService {
+  private dockerService: DockerService
+  private manifestService: CollectionManifestService
+
+  constructor(dockerService?: DockerService, manifestService?: CollectionManifestService) {
+    this.dockerService = dockerService ?? new DockerService()
+    this.manifestService = manifestService ?? new CollectionManifestService()
+  }
+
+  private get workerBase(): string {
+    return (env.get('CREATOR_PACKS_WORKER_BASE') || DEFAULT_WORKER_BASE).replace(/\/+$/, '')
+  }
+
+  /** Stable, per-version download URL. Not signed — the gate is the auth header. */
+  packUrl(id: string, version: string): string {
+    return `${this.workerBase}/packs/${id}_${version}.zim`
+  }
+
+  async listPacksWithStatus(): Promise<CreatorPackWithStatus[]> {
+    return this.manifestService.getCreatorPacksWithStatus()
+  }
+
+  /**
+   * Install a pack by catalog id: ensure Kiwix is present (auto-install if not),
+   * then dispatch the gated ZIM download onto the shared rail.
+   */
+  async installPack(packId: string): Promise<InstallPackResult> {
+    const appKey = env.get('CREATOR_PACKS_APP_KEY')
+    if (!appKey) {
+      logger.error('[CreatorPackService] CREATOR_PACKS_APP_KEY is not set; cannot install packs')
+      return { code: 'not_configured' }
+    }
+
+    const spec = await this.manifestService.getSpecWithFallback<CreatorPacksSpec>('creator_packs')
+    const pack = spec?.packs.find((p) => p.id === packId)
+    if (!pack) {
+      return { code: 'not_found' }
+    }
+
+    const { default: InstalledResource } = await import('#models/installed_resource')
+    const existing = await InstalledResource.query()
+      .where('resource_type', 'zim')
+      .where('resource_id', pack.resource_id)
+      .first()
+    if (existing && existing.version === pack.version) {
+      return { code: 'already_installed' }
+    }
+
+    const url = this.packUrl(pack.id, pack.version)
+    if (await RunDownloadJob.getActiveByUrl(url)) {
+      return { code: 'already_downloading' }
+    }
+
+    // Kiwix is a hard prerequisite — it's what serves the pack. Auto-install it if
+    // absent (its own preinstall bootstraps a mini-wiki ZIM so the container can
+    // start). Best-effort: if the install can't be kicked off we still download the
+    // pack so the bytes are on disk; Kiwix hot-reloads it once installed.
+    await this.ensureKiwixInstalled()
+
+    const filename = `${pack.id}_${pack.version}.zim`
+    const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename)
+
+    await RunDownloadJob.dispatch({
+      url,
+      filepath,
+      timeout: 30000,
+      allowedMimeTypes: ZIM_MIME_TYPES,
+      forceNew: true,
+      filetype: 'zim',
+      title: pack.name,
+      totalBytes: pack.size_mb ? pack.size_mb * 1024 * 1024 : undefined,
+      requestHeaders: { Authorization: `Bearer ${appKey}` },
+      resourceMetadata: {
+        resource_id: pack.resource_id,
+        version: pack.version,
+        collection_ref: 'creator-packs',
+        skip_embedding: true,
+      },
+    })
+
+    logger.info(`[CreatorPackService] Dispatched download for pack ${pack.id} (${filename})`)
+    return { code: 'dispatched', filename }
+  }
+
+  private async ensureKiwixInstalled(): Promise<void> {
+    try {
+      const kiwixUrl = await this.dockerService.getServiceURL(SERVICE_NAMES.KIWIX)
+      if (kiwixUrl) return // already installed and resolvable
+
+      logger.info('[CreatorPackService] Kiwix not installed; auto-installing before pack download')
+      const result = await this.dockerService.createContainerPreflight(SERVICE_NAMES.KIWIX)
+      if (!result.success) {
+        // e.g. "already installing" — not fatal, the pack download proceeds regardless.
+        logger.warn(`[CreatorPackService] Kiwix preflight did not start: ${result.message}`)
+      }
+    } catch (error: any) {
+      logger.warn(
+        `[CreatorPackService] Kiwix auto-install failed (continuing with pack download): ${error?.message || error}`
+      )
+    }
+  }
+}

admin/app/utils/downloads.ts

@@ -24,6 +24,7 @@ export async function doResumableDownload({
   onComplete,
   forceNew = false,
   allowedMimeTypes,
+  requestHeaders,
 }: DoResumableDownloadParams): Promise<string> {
   const dirname = path.dirname(filepath)
   await ensureDirectoryExists(dirname)
@@ -41,10 +42,12 @@ export async function doResumableDownload({
     appendMode = true
   }
 
-  // Get file info with HEAD request first
+  // Get file info with HEAD request first. Gated sources (Creator Packs) require
+  // the auth header on the HEAD too, or the probe 401s before the GET is reached.
   const headResponse = await axios.head(url, {
     signal,
     timeout,
+    headers: requestHeaders,
   })
 
   // Some upstream hosts (notably download.kiwix.org for .zim files) don't set a
@@ -88,7 +91,10 @@ export async function doResumableDownload({
     appendMode = false
   }
 
-  const headers: Record<string, string> = {}
+  // Seed with any caller-supplied headers (e.g. Creator Packs' Authorization
+  // bearer) so every GET attempt below — including the range-restart retry —
+  // carries them. The Range header composes on top.
+  const headers: Record<string, string> = { ...requestHeaders }
   if (supportsRangeRequests && startByte > 0) {
     headers.Range = `bytes=${startByte}-`
   }

admin/app/validators/curated_collections.ts

@@ -68,6 +68,30 @@ export const wikipediaSpecSchema = vine.object({
   ).minLength(1),
 })
 
+// ---- Creator Packs spec (versioned) ----
+//
+// Display metadata only — deliberately NO `url` field. The ZIM bytes are resolved
+// through the entitlement Worker at install time (CreatorPackService), never from
+// this public catalog. See project_content_creator_packs.
+export const creatorPacksSpecSchema = vine.object({
+  spec_version: vine.string(),
+  packs: vine.array(
+    vine.object({
+      id: vine.string(),
+      name: vine.string(),
+      creator: vine.string(),
+      description: vine.string(),
+      version: vine.string(),
+      resource_id: vine.string(),
+      video_count: vine.number().min(0),
+      size_mb: vine.number().min(0),
+      license_id: vine.string(),
+      poster_url: vine.string().url().optional(),
+      logo_url: vine.string().url().optional(),
+    })
+  ).minLength(1),
+})
+
 // ---- Wikipedia validators (used by ZimService) ----
 
 export const wikipediaOptionSchema = vine.object({

admin/start/env.ts

@@ -62,4 +62,18 @@ export default await Env.create(new URL('../', import.meta.url), {
   |----------------------------------------------------------
   */
   NOMAD_API_URL: Env.schema.string.optional(),
+
+  /*
+  |----------------------------------------------------------
+  | Creator Packs (gated content downloads)
+  |----------------------------------------------------------
+  | CREATOR_PACKS_APP_KEY is the shared bearer key the entitlement Worker
+  | requires; official release builds inject it at build time (it is NOT
+  | committed to source). When unset, pack installs fail with a clear
+  | "not configured" error and the rest of the app is unaffected.
+  | CREATOR_PACKS_WORKER_BASE overrides the default Worker origin (e.g. to
+  | point at a branded packs.projectnomad.us domain later).
+  */
+  CREATOR_PACKS_APP_KEY: Env.schema.string.optional(),
+  CREATOR_PACKS_WORKER_BASE: Env.schema.string.optional(),
 })

admin/start/routes.ts

@@ -19,6 +19,7 @@ import SettingsController from '#controllers/settings_controller'
 import SupplyDepotController from '#controllers/supply_depot_controller'
 import SystemController from '#controllers/system_controller'
 import CollectionUpdatesController from '#controllers/collection_updates_controller'
+import CreatorPacksController from '#controllers/creator_packs_controller'
 import ZimController from '#controllers/zim_controller'
 import router from '@adonisjs/core/services/router'
 import transmit from '@adonisjs/transmit/services/main'
@@ -225,6 +226,13 @@ router
   })
   .prefix('/api/zim')
 
+router
+  .group(() => {
+    router.get('/', [CreatorPacksController, 'index'])
+    router.post('/:id/install', [CreatorPacksController, 'install'])
+  })
+  .prefix('/api/creator-packs')
+
 router
   .group(() => {
     router.post('/run', [BenchmarkController, 'run'])

admin/types/collections.ts

@@ -58,7 +58,45 @@ export type WikipediaSpec = {
   options: WikipediaOption[]
 }
 
-export type ManifestType = 'zim_categories' | 'maps' | 'wikipedia'
+// ---- Creator Packs spec ----
+//
+// A creator pack is a branded ZIM of a creator's curated YouTube videos, served
+// offline through Kiwix. The public catalog carries display metadata ONLY — there
+// is NO download url here. The bytes live in a private R2 bucket behind an
+// entitlement Worker; CreatorPackService resolves the (stable) Worker URL and
+// sends the auth header at install time. See project_content_creator_packs.
+export type CreatorPack = {
+  id: string
+  name: string
+  creator: string
+  description: string
+  /** YYYY-MM; combines with id as `<id>_<version>.zim` (parseZimFilename). */
+  version: string
+  /** InstalledResource.resource_id used for install/status reconciliation. */
+  resource_id: string
+  video_count: number
+  size_mb: number
+  license_id: string
+  /** Optional card art (Phase 2). The Kiwix grid icon is baked into the ZIM. */
+  poster_url?: string
+  logo_url?: string
+}
+
+export type CreatorPacksSpec = {
+  spec_version: string
+  packs: CreatorPack[]
+}
+
+export type CreatorPackStatus = 'installed' | 'downloading' | 'available'
+
+export type CreatorPackWithStatus = CreatorPack & {
+  status: CreatorPackStatus
+  installed_version?: string
+  /** Set when an installed pack has a newer catalog version available. */
+  available_update_version?: string
+}
+
+export type ManifestType = 'zim_categories' | 'maps' | 'wikipedia' | 'creator_packs'
 
 export type ResourceStatus = 'installed' | 'not_installed' | 'update_available'
 

admin/types/downloads.ts

@@ -7,6 +7,13 @@ export type DoResumableDownloadParams = {
   onProgress?: (progress: DoResumableDownloadProgress) => void
   onComplete?: (url: string, path: string) => void | Promise<void>
   forceNew?: boolean
+  /**
+   * Extra HTTP request headers sent on BOTH the HEAD probe and the GET stream.
+   * Used by Creator Packs to carry the `Authorization: Bearer <APP_KEY>` the
+   * entitlement Worker requires; the Range header still composes on top for
+   * resumed downloads. Kept generic so any gated source can reuse it.
+   */
+  requestHeaders?: Record<string, string>
 }
 
 export type DoResumableDownloadWithRetryParams = DoResumableDownloadParams & {
@@ -48,6 +55,13 @@ export type RunDownloadJobParams = Omit<
      * `failed` handler) so manual downloads never touch the counter.
      */
     auto?: boolean
+    /**
+     * Skip the RAG/knowledge-base ingestion branch in RunDownloadJob.onComplete.
+     * Set for video ZIMs (Creator Packs) — they are media galleries, not text,
+     * and must never be dispatched to EmbedFileJob or the replacement-reconcile
+     * path. The Kiwix rebuild still runs so the pack appears in the grid.
+     */
+    skip_embedding?: boolean
   }
 }
 

collections/creator-packs.json

@@ -0,0 +1,27 @@
+{
+  "spec_version": "1",
+  "packs": [
+    {
+      "id": "project-nomad",
+      "name": "Project NOMAD",
+      "creator": "Project NOMAD",
+      "description": "Official Project NOMAD videos, packaged for offline viewing on your NOMAD.",
+      "version": "2026-07",
+      "resource_id": "project-nomad",
+      "video_count": 5,
+      "size_mb": 178,
+      "license_id": "nomad-creator-pack-1.0"
+    },
+    {
+      "id": "crosstalk-solutions",
+      "name": "Crosstalk Solutions",
+      "creator": "Chris Sherwood",
+      "description": "Chris Sherwood's networking, UniFi & home lab videos, for offline NOMAD.",
+      "version": "2026-07",
+      "resource_id": "crosstalk-solutions",
+      "video_count": 30,
+      "size_mb": 2899,
+      "license_id": "nomad-creator-pack-1.0"
+    }
+  ]
+}