feat: support gated downloads for self-hosted curated content (#1172)

Adds an optional `auth: 'nomad_app_key'` discriminator to curated manifest
resources so a curated collection tier can carry content we host ourselves,
gated to official release builds. Without this, only Creator Packs could use
the entitlement Worker; curated tier installs always downloaded
unauthenticated.

No behaviour change for any existing manifest entry: absent `auth` means
unauthenticated, exactly as today.

- `auth` declared on both the type and the VineJS validator. It has to be on
  the validator or VineJS strips it silently on fetch, and the gated download
  would then go out with no header and 401 for everyone. A dedicated spec
  guards that regression.
- Gated resources are pinned to their manifest URL (resolveZimDownload skips
  the catalog comparison) and excluded from catalog update checks, so a
  resource-id collision cannot let a third-party mirror overwrite our content.
  Consequence, commented rather than implied: gated content does not
  auto-update; new versions ship via the manifest.
- 401/403 on a download now reports that an official build is required instead
  of a raw axios status, which is what a fork build will hit.
- The pure `isGatedResource` predicate is deliberately split from the
  env-reading header builder: importing `#start/env` into
  zim_download_resolution triggers env validation at import time and breaks its
  unit tests.

Reuses CREATOR_PACKS_APP_KEY rather than minting a second secret — the question
it answers ("is this an official build?") is identical for both content types.

Verified end to end on a test server: `auth` survives validation into the
cached spec, the Bearer header attaches to only the gated resource, the file
lands byte-exact with an installed_resources row and a Kiwix library entry, and
an entry with a gated URL but no `auth` field fails with the intended message.

No catalog entry is included here. Manifests are fetched live from `main`, so a
gated entry must not merge until this ships and is adopted — pre-`auth` builds
strip the field and 401.

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

admin/app/services/collection_manifest_service.ts

@@ -9,6 +9,7 @@ import WikipediaSelection from '#models/wikipedia_selection'
 import { QueueService } from './queue_service.js'
 import { RunDownloadJob } from '#jobs/run_download_job'
 import { zimCategoriesSpecSchema, mapsSpecSchema, wikipediaSpecSchema, creatorPacksSpecSchema } from '#validators/curated_collections'
+import { isGatedResource } from '../utils/hosted_content.js'
 import {
   ensureDirectoryExists,
   listDirectoryContents,
@@ -314,6 +315,39 @@ export class CollectionManifestService {
     })
   }
 
+  /**
+   * Resource ids in the ZIM manifest that we host ourselves behind the
+   * entitlement Worker (`auth: 'nomad_app_key'`).
+   *
+   * Used to keep gated content out of the Kiwix-catalog update path. Those
+   * resources are not in the openzim catalog, so they can never legitimately
+   * match there — but a resource-id collision would otherwise let a third-party
+   * mirror present itself as a newer version and overwrite our content. Their
+   * versions come from the manifest instead.
+   *
+   * Reads the CACHED spec rather than refetching: this sits on the scheduled
+   * update-check path and does not need a network round-trip. A gated resource
+   * cannot be installed without the manifest having been fetched first, so the
+   * cache is always populated by the time it matters.
+   *
+   * Returns an empty set if the manifest has never been cached, which correctly
+   * degrades to current behaviour rather than skipping every update.
+   */
+  async getGatedZimResourceIds(): Promise<Set<string>> {
+    const ids = new Set<string>()
+    const spec = await this.getCachedSpec<ZimCategoriesSpec>('zim_categories')
+    if (!spec) return ids
+
+    for (const category of spec.categories) {
+      for (const tier of category.tiers) {
+        for (const resource of tier.resources) {
+          if (isGatedResource(resource)) ids.add(resource.id)
+        }
+      }
+    }
+    return ids
+  }
+
   // ---- Tier resolution ----
 
   static resolveTierResources(tier: SpecTier, allTiers: SpecTier[]): SpecResource[] {

admin/app/services/collection_update_service.ts

@@ -10,6 +10,7 @@ import type {
   ContentUpdateCheckResult,
 } from '../../types/collections.js'
 import { KiwixCatalogService, reconcileResourceUpdateState } from './kiwix_catalog_service.js'
+import { CollectionManifestService } from './collection_manifest_service.js'
 
 const MAP_STORAGE_PATH = '/storage/maps'
 
@@ -27,7 +28,18 @@ export class CollectionUpdateService {
     // ZIM/map catalog update path only — exclude `dataset` resources (e.g. the
     // FDA drug labels), which are not filename-versioned and get their own
     // freshness path. No-op today (no dataset rows are written in this slice).
-    const installed = await InstalledResource.query().whereNot('resource_type', 'dataset')
+    const allInstalled = await InstalledResource.query().whereNot('resource_type', 'dataset')
+
+    // Content we host ourselves is versioned by the manifest, not by the Kiwix
+    // catalog, so it has no business in this check. Excluding it also means a
+    // resource-id collision can't let a third-party mirror present itself as a
+    // newer version and overwrite our content. See resolveZimDownload, which
+    // pins the same resources to their manifest URL on the install path.
+    const gatedIds = await new CollectionManifestService().getGatedZimResourceIds()
+    const installed = allInstalled.filter(
+      (r) => !(r.resource_type === 'zim' && gatedIds.has(r.resource_id))
+    )
+
     if (installed.length === 0) {
       return {
         updates: [],

admin/app/services/zim_service.ts

@@ -37,6 +37,7 @@ import type { CategoryWithStatus } from '../../types/collections.js'
 import CustomLibrarySource from '#models/custom_library_source'
 import { assertNotPrivateUrl } from '#validators/common'
 import { resolveZimDownload } from '../utils/zim_download_resolution.js'
+import { getHostedContentHeaders } from '../utils/hosted_content_auth.js'
 
 const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
 const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json'
@@ -334,6 +335,8 @@ export class ZimService {
         filetype: 'zim',
         title: (resource as any).title || undefined,
         totalBytes: resolved.sizeBytes,
+        // Undefined for every ungated resource, so the existing flow is untouched.
+        requestHeaders: getHostedContentHeaders(resource),
         resourceMetadata: {
           resource_id: resource.id,
           version: resolved.version,

admin/app/utils/downloads.ts

@@ -57,11 +57,28 @@ export async function doResumableDownload({
 
   // Get file info with HEAD request first. Gated sources (Creator Packs) require
   // the auth header on the HEAD too, or the probe 401s before the GET is reached.
-  const headResponse = await axios.head(url, {
-    signal,
-    timeout,
-    headers, 
-  })
+  let headResponse
+  try {
+    headResponse = await axios.head(url, {
+      signal,
+      timeout,
+      headers,
+    })
+  } catch (error: any) {
+    // A 401/403 from a gated source is not a network problem and the raw axios
+    // message ("Request failed with status code 401") reads like our server is
+    // broken. Translate it, because the actual cause is almost always a build
+    // without the entitlement key baked in — i.e. not an official release.
+    // failedReason is surfaced verbatim on the downloads UI.
+    const status = error?.response?.status
+    if (status === 401 || status === 403) {
+      throw new Error(
+        'This content is hosted by Project NOMAD and requires an official release build. ' +
+          `The download server rejected this install's credentials (HTTP ${status}).`
+      )
+    }
+    throw error
+  }
 
   // Some upstream hosts (notably download.kiwix.org for .zim files) don't set a
   // Content-Type header at all. Per RFC 7231 §3.1.1.5, "if no Content-Type is

admin/app/utils/hosted_content.ts

@@ -0,0 +1,17 @@
+import type { SpecResource } from '../../types/collections.js'
+
+/**
+ * Pure predicate for "is this a resource we host behind the entitlement Worker?"
+ *
+ * Deliberately kept free of any `#start/env` import. `zim_download_resolution` is
+ * a pure, unit-tested module, and importing the env-reading side of this (see
+ * hosted_content_auth.ts) would trigger env validation at module load and break
+ * those tests outside a configured app context.
+ */
+
+/** The only gating scheme we support today. See SpecResource.auth. */
+export const NOMAD_APP_KEY_AUTH = 'nomad_app_key' as const
+
+export function isGatedResource(resource: Pick<SpecResource, 'auth'>): boolean {
+  return resource.auth === NOMAD_APP_KEY_AUTH
+}

admin/app/utils/hosted_content_auth.ts

@@ -0,0 +1,38 @@
+import env from '#start/env'
+import type { SpecResource } from '../../types/collections.js'
+import { isGatedResource } from './hosted_content.js'
+
+/**
+ * Auth for curated content that WE host and pay egress for.
+ *
+ * Content we host sits in a private R2 bucket behind the entitlement Worker,
+ * which requires a bearer key that only official release builds bake in (see the
+ * Dockerfile ARG/ENV pair, fed from the CI secret). That is the whole point: a
+ * fork rebuilt from source cannot point at our bucket and spend our bandwidth.
+ *
+ * A manifest resource opts in with `auth: 'nomad_app_key'`. Everything else keeps
+ * downloading unauthenticated exactly as before.
+ *
+ * Note on the key name: this deliberately reuses CREATOR_PACKS_APP_KEY rather
+ * than minting a second secret. The question it answers ("is this an official
+ * build?") is identical for Creator Packs and for our own hosted content, so a
+ * second CI secret plus a second Dockerfile ARG would be real cost for no
+ * security gain. The name is narrower than the use; this comment is cheaper than
+ * the churn of renaming it across CI, the Dockerfile and the Worker.
+ *
+ * The pure `isGatedResource` predicate lives in hosted_content.ts so that
+ * modules which must not pull in env validation can still use it.
+ */
+export function getHostedContentHeaders(
+  resource: Pick<SpecResource, 'auth'>
+): Record<string, string> | undefined {
+  if (!isGatedResource(resource)) return undefined
+
+  const appKey = env.get('CREATOR_PACKS_APP_KEY')
+  if (!appKey) return undefined
+
+  // Deliberately still dispatches with no header when the key is absent: the
+  // Worker answers 401 and the download surfaces "official release build
+  // required", which is a more useful signal than a silent no-op.
+  return { Authorization: `Bearer ${appKey}` }
+}

admin/app/utils/zim_download_resolution.ts

@@ -1,5 +1,6 @@
 import type { CatalogResult } from '../services/kiwix_catalog_service.js'
 import type { SpecResource } from '../../types/collections.js'
+import { isGatedResource } from './hosted_content.js'
 
 export type ResolvedZimDownload = {
   url: string
@@ -31,6 +32,22 @@ export function resolveZimDownload(
 ): ResolvedZimDownload {
   const manifestSizeBytes = resource.size_mb > 0 ? resource.size_mb * 1024 * 1024 : undefined
 
+  // Content we host ourselves is pinned to the manifest URL, never the Kiwix
+  // catalog. It isn't in the openzim catalog at all, so this is normally a no-op
+  // — but a resource-id collision would otherwise silently redirect a gated
+  // download to a third-party mirror, losing both the auth header and any
+  // guarantee about what the bytes are.
+  //
+  // Consequence, stated rather than implied: gated content does NOT participate
+  // in catalog-driven auto-update. New versions ship by bumping the manifest.
+  if (isGatedResource(resource)) {
+    return {
+      url: resource.url,
+      version: resource.version,
+      sizeBytes: manifestSizeBytes,
+    }
+  }
+
   if (!latest || compareZimVersions(latest.version, resource.version) < 0) {
     return {
       url: resource.url,

admin/app/validators/curated_collections.ts

@@ -12,6 +12,11 @@ export const specResourceValidator = vine.object({
   // Resource-type discriminator (absent == 'zim'). Required here because VineJS
   // strips unknown keys, which would silently drop the field on manifest fetch.
   type: vine.enum(['zim', 'dataset']).optional(),
+  // Gated-download discriminator (absent == unauthenticated). Declared here for
+  // the same reason as `type`: VineJS strips unknown keys, so omitting it would
+  // silently drop the field on manifest fetch and every gated download would go
+  // out with no Authorization header and 401.
+  auth: vine.enum(['nomad_app_key']).optional(),
 })
 
 // ---- ZIM Categories spec (versioned) ----

admin/tests/unit/curated_resource_schema.spec.ts

@@ -0,0 +1,90 @@
+import * as assert from 'node:assert/strict'
+import { test } from 'node:test'
+import vine from '@vinejs/vine'
+
+import { zimCategoriesSpecSchema } from '../../app/validators/curated_collections.js'
+
+/**
+ * VineJS STRIPS unknown keys rather than rejecting them, so a manifest field that
+ * is not declared on the validator disappears silently on fetch. For `auth` that
+ * failure is invisible and expensive: the gated download would go out with no
+ * Authorization header and 401 for every user.
+ *
+ * These tests exist to catch that specific regression, so treat a failure here as
+ * "the validator lost a field", not "the test is wrong".
+ */
+
+function specWithResource(resource: Record<string, unknown>) {
+  return {
+    spec_version: '1.0.0',
+    categories: [
+      {
+        name: 'Survival & Preparedness',
+        slug: 'survival-and-preparedness',
+        icon: 'IconTent',
+        description: 'Field references for austere conditions',
+        language: 'en',
+        tiers: [
+          {
+            name: 'Comprehensive',
+            slug: 'comprehensive',
+            description: 'Everything we have',
+            resources: [resource],
+          },
+        ],
+      },
+    ],
+  }
+}
+
+const baseResource = {
+  id: 'field-manuals',
+  version: '2026-07',
+  title: 'US Military Field Manuals',
+  description: 'Public-domain US military field manuals',
+  url: 'https://nomad-packs-worker.chris-556.workers.dev/content/field-manuals_2026-07.zim',
+  size_mb: 2000,
+}
+
+test('auth survives manifest validation', async () => {
+  const validated: any = await vine.validate({
+    schema: zimCategoriesSpecSchema,
+    data: specWithResource({ ...baseResource, auth: 'nomad_app_key' }),
+  })
+
+  const resource = validated.categories[0].tiers[0].resources[0]
+  assert.equal(
+    resource.auth,
+    'nomad_app_key',
+    'auth was stripped by the validator — gated downloads would 401'
+  )
+})
+
+test('a resource without auth validates and reports auth as undefined', async () => {
+  const validated: any = await vine.validate({
+    schema: zimCategoriesSpecSchema,
+    data: specWithResource(baseResource),
+  })
+
+  assert.equal(validated.categories[0].tiers[0].resources[0].auth, undefined)
+})
+
+test('an unrecognised auth scheme is rejected rather than silently ignored', async () => {
+  await assert.rejects(() =>
+    vine.validate({
+      schema: zimCategoriesSpecSchema,
+      data: specWithResource({ ...baseResource, auth: 'something_else' }),
+    })
+  )
+})
+
+test('auth and type coexist on one resource', async () => {
+  const validated: any = await vine.validate({
+    schema: zimCategoriesSpecSchema,
+    data: specWithResource({ ...baseResource, type: 'zim', auth: 'nomad_app_key' }),
+  })
+
+  const resource = validated.categories[0].tiers[0].resources[0]
+  assert.equal(resource.type, 'zim')
+  assert.equal(resource.auth, 'nomad_app_key')
+})

admin/tests/unit/zim_download_resolution.spec.ts

@@ -73,3 +73,55 @@ test('non-padded catalog months are compared numerically', () => {
   )
   assert.equal(resolved.version, '2026-10')
 })
+
+// ---- Gated, self-hosted content ----
+//
+// Resources we host behind the entitlement Worker are pinned to the manifest URL.
+// They are not in the openzim catalog, so a catalog match can only ever be a
+// resource-id collision, and following it would swap our content for a third
+// party's AND drop the Authorization header.
+
+const gatedResource = {
+  id: 'field-manuals',
+  version: '2026-07',
+  title: 'US Military Field Manuals',
+  description: 'Public-domain US military field manuals',
+  url: 'https://nomad-packs-worker.chris-556.workers.dev/content/field-manuals_2026-07.zim',
+  size_mb: 2_000,
+  auth: 'nomad_app_key' as const,
+}
+
+test('gated resource ignores a newer catalog result and stays on the manifest URL', () => {
+  const resolved = resolveZimDownload(gatedResource, {
+    version: '2026-12',
+    download_url: 'https://download.kiwix.org/zim/other/field-manuals_2026-12.zim',
+    size_bytes: 9_999_999,
+  })
+
+  assert.deepEqual(resolved, {
+    url: gatedResource.url,
+    version: gatedResource.version,
+    sizeBytes: gatedResource.size_mb * 1024 * 1024,
+  })
+})
+
+test('gated resource resolves normally with no catalog result', () => {
+  assert.deepEqual(resolveZimDownload(gatedResource, null), {
+    url: gatedResource.url,
+    version: gatedResource.version,
+    sizeBytes: gatedResource.size_mb * 1024 * 1024,
+  })
+})
+
+test('absent auth leaves catalog precedence untouched', () => {
+  const resolved = resolveZimDownload(manifestResource, {
+    version: '2026-06',
+    download_url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim',
+    size_bytes: 12_531_944_448,
+  })
+
+  assert.equal(
+    resolved.url,
+    'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim'
+  )
+})

admin/types/collections.ts

@@ -11,6 +11,19 @@ export type SpecResource = {
    * installer to the DB-ingested drug pipeline instead.
    */
   type?: 'zim' | 'dataset'
+  /**
+   * Marks a resource we host ourselves behind the entitlement Worker. Absent ==
+   * unauthenticated, so every existing manifest entry is unchanged.
+   *
+   * 'nomad_app_key' means "send the bearer key that official release builds bake
+   * in". It also pins the download to `url`: see resolveZimDownload, which
+   * deliberately skips the Kiwix-catalog comparison for these so a resource-id
+   * collision can never redirect our gated content to a third-party mirror.
+   *
+   * An enum rather than a boolean so a second scheme can be added later without
+   * another schema change.
+   */
+  auth?: 'nomad_app_key'
 }
 
 export type SpecTier = {