diff --git a/admin/app/services/collection_manifest_service.ts b/admin/app/services/collection_manifest_service.ts index 703d14e..967a543 100644 --- a/admin/app/services/collection_manifest_service.ts +++ b/admin/app/services/collection_manifest_service.ts @@ -9,6 +9,7 @@ import WikipediaSelection from '#models/wikipedia_selection' import { QueueService } from './queue_service.js' import { RunDownloadJob } from '#jobs/run_download_job' import { zimCategoriesSpecSchema, mapsSpecSchema, wikipediaSpecSchema, creatorPacksSpecSchema } from '#validators/curated_collections' +import { isGatedResource } from '../utils/hosted_content.js' import { ensureDirectoryExists, listDirectoryContents, @@ -314,6 +315,39 @@ export class CollectionManifestService { }) } + /** + * Resource ids in the ZIM manifest that we host ourselves behind the + * entitlement Worker (`auth: 'nomad_app_key'`). + * + * Used to keep gated content out of the Kiwix-catalog update path. Those + * resources are not in the openzim catalog, so they can never legitimately + * match there — but a resource-id collision would otherwise let a third-party + * mirror present itself as a newer version and overwrite our content. Their + * versions come from the manifest instead. + * + * Reads the CACHED spec rather than refetching: this sits on the scheduled + * update-check path and does not need a network round-trip. A gated resource + * cannot be installed without the manifest having been fetched first, so the + * cache is always populated by the time it matters. + * + * Returns an empty set if the manifest has never been cached, which correctly + * degrades to current behaviour rather than skipping every update. + */ + async getGatedZimResourceIds(): Promise> { + const ids = new Set() + const spec = await this.getCachedSpec('zim_categories') + if (!spec) return ids + + for (const category of spec.categories) { + for (const tier of category.tiers) { + for (const resource of tier.resources) { + if (isGatedResource(resource)) ids.add(resource.id) + } + } + } + return ids + } + // ---- Tier resolution ---- static resolveTierResources(tier: SpecTier, allTiers: SpecTier[]): SpecResource[] { diff --git a/admin/app/services/collection_update_service.ts b/admin/app/services/collection_update_service.ts index 8800bba..ea497b2 100644 --- a/admin/app/services/collection_update_service.ts +++ b/admin/app/services/collection_update_service.ts @@ -10,6 +10,7 @@ import type { ContentUpdateCheckResult, } from '../../types/collections.js' import { KiwixCatalogService, reconcileResourceUpdateState } from './kiwix_catalog_service.js' +import { CollectionManifestService } from './collection_manifest_service.js' const MAP_STORAGE_PATH = '/storage/maps' @@ -27,7 +28,18 @@ export class CollectionUpdateService { // ZIM/map catalog update path only — exclude `dataset` resources (e.g. the // FDA drug labels), which are not filename-versioned and get their own // freshness path. No-op today (no dataset rows are written in this slice). - const installed = await InstalledResource.query().whereNot('resource_type', 'dataset') + const allInstalled = await InstalledResource.query().whereNot('resource_type', 'dataset') + + // Content we host ourselves is versioned by the manifest, not by the Kiwix + // catalog, so it has no business in this check. Excluding it also means a + // resource-id collision can't let a third-party mirror present itself as a + // newer version and overwrite our content. See resolveZimDownload, which + // pins the same resources to their manifest URL on the install path. + const gatedIds = await new CollectionManifestService().getGatedZimResourceIds() + const installed = allInstalled.filter( + (r) => !(r.resource_type === 'zim' && gatedIds.has(r.resource_id)) + ) + if (installed.length === 0) { return { updates: [], diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index a7cd149..2de6a36 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -37,6 +37,7 @@ import type { CategoryWithStatus } from '../../types/collections.js' import CustomLibrarySource from '#models/custom_library_source' import { assertNotPrivateUrl } from '#validators/common' import { resolveZimDownload } from '../utils/zim_download_resolution.js' +import { getHostedContentHeaders } from '../utils/hosted_content_auth.js' const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream'] const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json' @@ -334,6 +335,8 @@ export class ZimService { filetype: 'zim', title: (resource as any).title || undefined, totalBytes: resolved.sizeBytes, + // Undefined for every ungated resource, so the existing flow is untouched. + requestHeaders: getHostedContentHeaders(resource), resourceMetadata: { resource_id: resource.id, version: resolved.version, diff --git a/admin/app/utils/downloads.ts b/admin/app/utils/downloads.ts index ae92b7b..cf60f73 100644 --- a/admin/app/utils/downloads.ts +++ b/admin/app/utils/downloads.ts @@ -57,11 +57,28 @@ export async function doResumableDownload({ // Get file info with HEAD request first. Gated sources (Creator Packs) require // the auth header on the HEAD too, or the probe 401s before the GET is reached. - const headResponse = await axios.head(url, { - signal, - timeout, - headers, - }) + let headResponse + try { + headResponse = await axios.head(url, { + signal, + timeout, + headers, + }) + } catch (error: any) { + // A 401/403 from a gated source is not a network problem and the raw axios + // message ("Request failed with status code 401") reads like our server is + // broken. Translate it, because the actual cause is almost always a build + // without the entitlement key baked in — i.e. not an official release. + // failedReason is surfaced verbatim on the downloads UI. + const status = error?.response?.status + if (status === 401 || status === 403) { + throw new Error( + 'This content is hosted by Project NOMAD and requires an official release build. ' + + `The download server rejected this install's credentials (HTTP ${status}).` + ) + } + throw error + } // Some upstream hosts (notably download.kiwix.org for .zim files) don't set a // Content-Type header at all. Per RFC 7231 §3.1.1.5, "if no Content-Type is diff --git a/admin/app/utils/hosted_content.ts b/admin/app/utils/hosted_content.ts new file mode 100644 index 0000000..6ac98aa --- /dev/null +++ b/admin/app/utils/hosted_content.ts @@ -0,0 +1,17 @@ +import type { SpecResource } from '../../types/collections.js' + +/** + * Pure predicate for "is this a resource we host behind the entitlement Worker?" + * + * Deliberately kept free of any `#start/env` import. `zim_download_resolution` is + * a pure, unit-tested module, and importing the env-reading side of this (see + * hosted_content_auth.ts) would trigger env validation at module load and break + * those tests outside a configured app context. + */ + +/** The only gating scheme we support today. See SpecResource.auth. */ +export const NOMAD_APP_KEY_AUTH = 'nomad_app_key' as const + +export function isGatedResource(resource: Pick): boolean { + return resource.auth === NOMAD_APP_KEY_AUTH +} diff --git a/admin/app/utils/hosted_content_auth.ts b/admin/app/utils/hosted_content_auth.ts new file mode 100644 index 0000000..2281408 --- /dev/null +++ b/admin/app/utils/hosted_content_auth.ts @@ -0,0 +1,38 @@ +import env from '#start/env' +import type { SpecResource } from '../../types/collections.js' +import { isGatedResource } from './hosted_content.js' + +/** + * Auth for curated content that WE host and pay egress for. + * + * Content we host sits in a private R2 bucket behind the entitlement Worker, + * which requires a bearer key that only official release builds bake in (see the + * Dockerfile ARG/ENV pair, fed from the CI secret). That is the whole point: a + * fork rebuilt from source cannot point at our bucket and spend our bandwidth. + * + * A manifest resource opts in with `auth: 'nomad_app_key'`. Everything else keeps + * downloading unauthenticated exactly as before. + * + * Note on the key name: this deliberately reuses CREATOR_PACKS_APP_KEY rather + * than minting a second secret. The question it answers ("is this an official + * build?") is identical for Creator Packs and for our own hosted content, so a + * second CI secret plus a second Dockerfile ARG would be real cost for no + * security gain. The name is narrower than the use; this comment is cheaper than + * the churn of renaming it across CI, the Dockerfile and the Worker. + * + * The pure `isGatedResource` predicate lives in hosted_content.ts so that + * modules which must not pull in env validation can still use it. + */ +export function getHostedContentHeaders( + resource: Pick +): Record | undefined { + if (!isGatedResource(resource)) return undefined + + const appKey = env.get('CREATOR_PACKS_APP_KEY') + if (!appKey) return undefined + + // Deliberately still dispatches with no header when the key is absent: the + // Worker answers 401 and the download surfaces "official release build + // required", which is a more useful signal than a silent no-op. + return { Authorization: `Bearer ${appKey}` } +} diff --git a/admin/app/utils/zim_download_resolution.ts b/admin/app/utils/zim_download_resolution.ts index 9817453..af15ba6 100644 --- a/admin/app/utils/zim_download_resolution.ts +++ b/admin/app/utils/zim_download_resolution.ts @@ -1,5 +1,6 @@ import type { CatalogResult } from '../services/kiwix_catalog_service.js' import type { SpecResource } from '../../types/collections.js' +import { isGatedResource } from './hosted_content.js' export type ResolvedZimDownload = { url: string @@ -31,6 +32,22 @@ export function resolveZimDownload( ): ResolvedZimDownload { const manifestSizeBytes = resource.size_mb > 0 ? resource.size_mb * 1024 * 1024 : undefined + // Content we host ourselves is pinned to the manifest URL, never the Kiwix + // catalog. It isn't in the openzim catalog at all, so this is normally a no-op + // — but a resource-id collision would otherwise silently redirect a gated + // download to a third-party mirror, losing both the auth header and any + // guarantee about what the bytes are. + // + // Consequence, stated rather than implied: gated content does NOT participate + // in catalog-driven auto-update. New versions ship by bumping the manifest. + if (isGatedResource(resource)) { + return { + url: resource.url, + version: resource.version, + sizeBytes: manifestSizeBytes, + } + } + if (!latest || compareZimVersions(latest.version, resource.version) < 0) { return { url: resource.url, diff --git a/admin/app/validators/curated_collections.ts b/admin/app/validators/curated_collections.ts index d0e4f59..8d1ec58 100644 --- a/admin/app/validators/curated_collections.ts +++ b/admin/app/validators/curated_collections.ts @@ -12,6 +12,11 @@ export const specResourceValidator = vine.object({ // Resource-type discriminator (absent == 'zim'). Required here because VineJS // strips unknown keys, which would silently drop the field on manifest fetch. type: vine.enum(['zim', 'dataset']).optional(), + // Gated-download discriminator (absent == unauthenticated). Declared here for + // the same reason as `type`: VineJS strips unknown keys, so omitting it would + // silently drop the field on manifest fetch and every gated download would go + // out with no Authorization header and 401. + auth: vine.enum(['nomad_app_key']).optional(), }) // ---- ZIM Categories spec (versioned) ---- diff --git a/admin/tests/unit/curated_resource_schema.spec.ts b/admin/tests/unit/curated_resource_schema.spec.ts new file mode 100644 index 0000000..cf3e2ba --- /dev/null +++ b/admin/tests/unit/curated_resource_schema.spec.ts @@ -0,0 +1,90 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' +import vine from '@vinejs/vine' + +import { zimCategoriesSpecSchema } from '../../app/validators/curated_collections.js' + +/** + * VineJS STRIPS unknown keys rather than rejecting them, so a manifest field that + * is not declared on the validator disappears silently on fetch. For `auth` that + * failure is invisible and expensive: the gated download would go out with no + * Authorization header and 401 for every user. + * + * These tests exist to catch that specific regression, so treat a failure here as + * "the validator lost a field", not "the test is wrong". + */ + +function specWithResource(resource: Record) { + return { + spec_version: '1.0.0', + categories: [ + { + name: 'Survival & Preparedness', + slug: 'survival-and-preparedness', + icon: 'IconTent', + description: 'Field references for austere conditions', + language: 'en', + tiers: [ + { + name: 'Comprehensive', + slug: 'comprehensive', + description: 'Everything we have', + resources: [resource], + }, + ], + }, + ], + } +} + +const baseResource = { + id: 'field-manuals', + version: '2026-07', + title: 'US Military Field Manuals', + description: 'Public-domain US military field manuals', + url: 'https://nomad-packs-worker.chris-556.workers.dev/content/field-manuals_2026-07.zim', + size_mb: 2000, +} + +test('auth survives manifest validation', async () => { + const validated: any = await vine.validate({ + schema: zimCategoriesSpecSchema, + data: specWithResource({ ...baseResource, auth: 'nomad_app_key' }), + }) + + const resource = validated.categories[0].tiers[0].resources[0] + assert.equal( + resource.auth, + 'nomad_app_key', + 'auth was stripped by the validator — gated downloads would 401' + ) +}) + +test('a resource without auth validates and reports auth as undefined', async () => { + const validated: any = await vine.validate({ + schema: zimCategoriesSpecSchema, + data: specWithResource(baseResource), + }) + + assert.equal(validated.categories[0].tiers[0].resources[0].auth, undefined) +}) + +test('an unrecognised auth scheme is rejected rather than silently ignored', async () => { + await assert.rejects(() => + vine.validate({ + schema: zimCategoriesSpecSchema, + data: specWithResource({ ...baseResource, auth: 'something_else' }), + }) + ) +}) + +test('auth and type coexist on one resource', async () => { + const validated: any = await vine.validate({ + schema: zimCategoriesSpecSchema, + data: specWithResource({ ...baseResource, type: 'zim', auth: 'nomad_app_key' }), + }) + + const resource = validated.categories[0].tiers[0].resources[0] + assert.equal(resource.type, 'zim') + assert.equal(resource.auth, 'nomad_app_key') +}) diff --git a/admin/tests/unit/zim_download_resolution.spec.ts b/admin/tests/unit/zim_download_resolution.spec.ts index 11f55ce..9009397 100644 --- a/admin/tests/unit/zim_download_resolution.spec.ts +++ b/admin/tests/unit/zim_download_resolution.spec.ts @@ -73,3 +73,55 @@ test('non-padded catalog months are compared numerically', () => { ) assert.equal(resolved.version, '2026-10') }) + +// ---- Gated, self-hosted content ---- +// +// Resources we host behind the entitlement Worker are pinned to the manifest URL. +// They are not in the openzim catalog, so a catalog match can only ever be a +// resource-id collision, and following it would swap our content for a third +// party's AND drop the Authorization header. + +const gatedResource = { + id: 'field-manuals', + version: '2026-07', + title: 'US Military Field Manuals', + description: 'Public-domain US military field manuals', + url: 'https://nomad-packs-worker.chris-556.workers.dev/content/field-manuals_2026-07.zim', + size_mb: 2_000, + auth: 'nomad_app_key' as const, +} + +test('gated resource ignores a newer catalog result and stays on the manifest URL', () => { + const resolved = resolveZimDownload(gatedResource, { + version: '2026-12', + download_url: 'https://download.kiwix.org/zim/other/field-manuals_2026-12.zim', + size_bytes: 9_999_999, + }) + + assert.deepEqual(resolved, { + url: gatedResource.url, + version: gatedResource.version, + sizeBytes: gatedResource.size_mb * 1024 * 1024, + }) +}) + +test('gated resource resolves normally with no catalog result', () => { + assert.deepEqual(resolveZimDownload(gatedResource, null), { + url: gatedResource.url, + version: gatedResource.version, + sizeBytes: gatedResource.size_mb * 1024 * 1024, + }) +}) + +test('absent auth leaves catalog precedence untouched', () => { + const resolved = resolveZimDownload(manifestResource, { + version: '2026-06', + download_url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim', + size_bytes: 12_531_944_448, + }) + + assert.equal( + resolved.url, + 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim' + ) +}) diff --git a/admin/types/collections.ts b/admin/types/collections.ts index 12522bb..771c5f0 100644 --- a/admin/types/collections.ts +++ b/admin/types/collections.ts @@ -11,6 +11,19 @@ export type SpecResource = { * installer to the DB-ingested drug pipeline instead. */ type?: 'zim' | 'dataset' + /** + * Marks a resource we host ourselves behind the entitlement Worker. Absent == + * unauthenticated, so every existing manifest entry is unchanged. + * + * 'nomad_app_key' means "send the bearer key that official release builds bake + * in". It also pins the download to `url`: see resolveZimDownload, which + * deliberately skips the Kiwix-catalog comparison for these so a resource-id + * collision can never redirect our gated content to a third-party mirror. + * + * An enum rather than a boolean so a second scheme can be added later without + * another schema change. + */ + auth?: 'nomad_app_key' } export type SpecTier = {