diff --git a/admin/app/models/installed_resource.ts b/admin/app/models/installed_resource.ts index 0cfe36b..dedb76b 100644 --- a/admin/app/models/installed_resource.ts +++ b/admin/app/models/installed_resource.ts @@ -11,7 +11,7 @@ export default class InstalledResource extends BaseModel { declare resource_id: string @column() - declare resource_type: 'zim' | 'map' + declare resource_type: 'zim' | 'map' | 'dataset' @column() declare collection_ref: string | null diff --git a/admin/app/services/collection_update_service.ts b/admin/app/services/collection_update_service.ts index 8b4e388..8800bba 100644 --- a/admin/app/services/collection_update_service.ts +++ b/admin/app/services/collection_update_service.ts @@ -24,7 +24,10 @@ export class CollectionUpdateService { * state (version + cool-off anchor) so the auto-updater can act on it later. */ async checkForUpdates(): Promise { - const installed = await InstalledResource.all() + // ZIM/map catalog update path only — exclude `dataset` resources (e.g. the + // FDA drug labels), which are not filename-versioned and get their own + // freshness path. No-op today (no dataset rows are written in this slice). + const installed = await InstalledResource.query().whereNot('resource_type', 'dataset') if (installed.length === 0) { return { updates: [], @@ -35,7 +38,11 @@ export class CollectionUpdateService { try { const catalog = new KiwixCatalogService() const latestByKey = await catalog.getLatestForResources( - installed.map((r) => ({ resource_id: r.resource_id, resource_type: r.resource_type })) + // `dataset` rows are filtered out above, so the type narrows to ZIM/map. + installed.map((r) => ({ + resource_id: r.resource_id, + resource_type: r.resource_type as 'zim' | 'map', + })) ) const now = DateTime.now() @@ -47,7 +54,7 @@ export class CollectionUpdateService { if (latest && latest.version > resource.version) { updates.push({ resource_id: resource.resource_id, - resource_type: resource.resource_type, + resource_type: resource.resource_type as 'zim' | 'map', installed_version: resource.version, latest_version: latest.version, download_url: latest.download_url, diff --git a/admin/app/services/content_auto_update_service.ts b/admin/app/services/content_auto_update_service.ts index fd798ab..ed2b32b 100644 --- a/admin/app/services/content_auto_update_service.ts +++ b/admin/app/services/content_auto_update_service.ts @@ -269,9 +269,15 @@ export class ContentAutoUpdateService { await this.maybeResetWindowBudget(config, now) // Local catalog check + persist available-update state for every resource. - const installed = await InstalledResource.all() + // ZIM/map catalog path only — `dataset` resources are excluded (they get + // their own freshness path; no-op today as none are written in this slice). + const installed = await InstalledResource.query().whereNot('resource_type', 'dataset') const latestByKey = await this.catalog.getLatestForResources( - installed.map((r) => ({ resource_id: r.resource_id, resource_type: r.resource_type })) + // `dataset` rows are filtered out above, so the type narrows to ZIM/map. + installed.map((r) => ({ + resource_id: r.resource_id, + resource_type: r.resource_type as 'zim' | 'map', + })) ) for (const resource of installed) { const latest = latestByKey.get(`${resource.resource_type}:${resource.resource_id}`) ?? null @@ -321,7 +327,8 @@ export class ContentAutoUpdateService { const result = await this.collectionUpdateService.applyUpdate( { resource_id: candidate.resource.resource_id, - resource_type: candidate.resource.resource_type, + // `dataset` rows are filtered out of `installed` above, so ZIM/map. + resource_type: candidate.resource.resource_type as 'zim' | 'map', installed_version: candidate.resource.version, latest_version: candidate.version, download_url: candidate.download_url, @@ -407,7 +414,9 @@ export class ContentAutoUpdateService { const now = overrides.now ?? DateTime.now() const withinWindow = isWithinWindow(config.windowStart, config.windowEnd, now) - const pending = await InstalledResource.query().whereNotNull('available_update_version') + const pending = await InstalledResource.query() + .whereNotNull('available_update_version') + .whereNot('resource_type', 'dataset') const eligible = pending.filter( (r) => this.resourceEligibility(r, config.cooloffHours, now).eligible ) @@ -506,7 +515,9 @@ export class ContentAutoUpdateService { const config = await this.getConfig() const now = DateTime.now() - const pending = await InstalledResource.query().whereNotNull('available_update_version') + const pending = await InstalledResource.query() + .whereNotNull('available_update_version') + .whereNot('resource_type', 'dataset') const resources: ContentAutoUpdateResourceStatus[] = pending.map((resource) => { const verdict = this.resourceEligibility(resource, config.cooloffHours, now) const size = resource.available_update_size_bytes ?? null @@ -514,7 +525,8 @@ export class ContentAutoUpdateService { config.maxBytesPerWindow > 0 && size !== null && size > config.maxBytesPerWindow return { resource_id: resource.resource_id, - resource_type: resource.resource_type, + // `dataset` rows are filtered out of `pending` above, so ZIM/map. + resource_type: resource.resource_type as 'zim' | 'map', current_version: resource.version, available_update_version: resource.available_update_version, size_bytes: size, diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 5a362e0..9b97338 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -27,6 +27,8 @@ import WikipediaSelection from '#models/wikipedia_selection' import InstalledResource from '#models/installed_resource' import CollectionManifest from '#models/collection_manifest' import { RunDownloadJob } from '#jobs/run_download_job' +import { DownloadDrugDataJob } from '#jobs/download_drug_data_job' +import { DrugReferenceService } from './drug_reference_service.js' import { SERVICE_NAMES } from '../../constants/service_names.js' import { CollectionManifestService } from './collection_manifest_service.js' import { KiwixLibraryService } from './kiwix_library_service.js' @@ -263,6 +265,30 @@ export class ZimService { const downloadFilenames: string[] = [] for (const resource of toDownload) { + // A `dataset` resource (e.g. the FDA drug labels) is DB-ingested, not a + // ZIM file — route it to the drug download+ingest pipeline instead of + // RunDownloadJob. The installed-filter above queries resource_type='zim', + // so a dataset is never seen as installed and would re-dispatch the ~1.7 GB + // download on every tier select. Guard against that with the existing + // drug-status signal: skip when the data is already present. + // (Planned follow-up: write an InstalledResource 'dataset' row on ingest + // 'ready' so the tier-status/badge math recognizes it uniformly — a + // maintainer decision still pending, out of scope for this slice.) + if (resource.type === 'dataset') { + const drugReferenceService = new DrugReferenceService() + const status = await drugReferenceService.getIngestStatus() + if (status.phase === 'ready' || status.rowCount > 0) { + logger.info('[ZimService] Drug dataset already ingested, skipping dispatch.') + continue + } + // DownloadDrugDataJob.dispatch() is idempotent on its deterministic + // jobId — a concurrent in-flight download returns "already running" + // without re-adding, so this is safe to call repeatedly. + await DownloadDrugDataJob.dispatch() + logger.info('[ZimService] Dispatched drug data download for dataset resource.') + continue + } + const existingJob = await RunDownloadJob.getActiveByUrl(resource.url) if (existingJob) { logger.warn(`[ZimService] Download already in progress for ${resource.url}, skipping.`) diff --git a/admin/app/validators/curated_collections.ts b/admin/app/validators/curated_collections.ts index af0d2b9..58c097a 100644 --- a/admin/app/validators/curated_collections.ts +++ b/admin/app/validators/curated_collections.ts @@ -9,6 +9,9 @@ export const specResourceValidator = vine.object({ description: vine.string(), url: vine.string().url(), size_mb: vine.number().min(0).optional(), + // Resource-type discriminator (absent == 'zim'). Required here because VineJS + // strips unknown keys, which would silently drop the field on manifest fetch. + type: vine.enum(['zim', 'dataset']).optional(), }) // ---- ZIM Categories spec (versioned) ---- diff --git a/admin/types/collections.ts b/admin/types/collections.ts index b8fbdb4..81e6024 100644 --- a/admin/types/collections.ts +++ b/admin/types/collections.ts @@ -5,6 +5,12 @@ export type SpecResource = { description: string url: string size_mb: number + /** + * Resource-type discriminator. Absent == 'zim' so every existing manifest + * entry keeps the current ZIM download path. 'dataset' routes the tier + * installer to the DB-ingested drug pipeline instead. + */ + type?: 'zim' | 'dataset' } export type SpecTier = {