RomanNum3ral
/
project-nomad
mirror of https://github.com/Crosstalk-Solutions/project-nomad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(collections): route a 'dataset' tier resource to the drug pipeline 

Add an optional `type` discriminator to SpecResource ('zim' | 'dataset',
absent == 'zim'), so the tier installer can carry a DB-ingested resource
alongside ZIM files. ZimService.downloadCategoryTier branches on it: a
'dataset' resource dispatches the existing FDA download+ingest pipeline
instead of RunDownloadJob, guarded against duplicate dispatch by the drug
ingest status. Every existing manifest entry has no `type` and keeps the
exact ZIM path.

Widen InstalledResource.resource_type to include 'dataset' and exclude
dataset rows from the ZIM/map catalog-update scan (datasets aren't
filename-versioned; their freshness path is separate). No dataset rows are
written yet: the InstalledResource 'dataset' row on ingest-ready, the
manifest entry, install-gating, and the downloads-aggregator integration
are follow-up commits.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Chris 2026-06-24 17:15:21 -05:00
parent b6ee5ed7bf
commit 754b0b1921
6 changed files with 64 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
admin/app/models/installed_resource.ts
View File

@ -11,7 +11,7 @@ export default class InstalledResource extends BaseModel {
declare resource_id: string
@column()
declare resource_type: 'zim' | 'map'
declare resource_type: 'zim' | 'map' | 'dataset'
@column()
declare collection_ref: string | null

13
admin/app/services/collection_update_service.ts
View File

@ -24,7 +24,10 @@ export class CollectionUpdateService {
* state (version + cool-off anchor) so the auto-updater can act on it later.
*/
async checkForUpdates(): Promise<ContentUpdateCheckResult> {
const installed = await InstalledResource.all()
// ZIM/map catalog update path only — exclude `dataset` resources (e.g. the
// FDA drug labels), which are not filename-versioned and get their own
// freshness path. No-op today (no dataset rows are written in this slice).
const installed = await InstalledResource.query().whereNot('resource_type', 'dataset')
if (installed.length === 0) {
return {
updates: [],
@ -35,7 +38,11 @@ export class CollectionUpdateService {
try {
const catalog = new KiwixCatalogService()
const latestByKey = await catalog.getLatestForResources(
installed.map((r) => ({ resource_id: r.resource_id, resource_type: r.resource_type }))
// `dataset` rows are filtered out above, so the type narrows to ZIM/map.
installed.map((r) => ({
resource_id: r.resource_id,
resource_type: r.resource_type as 'zim' | 'map',
}))
)
const now = DateTime.now()
@ -47,7 +54,7 @@ export class CollectionUpdateService {
if (latest && latest.version > resource.version) {
updates.push({
resource_id: resource.resource_id,
resource_type: resource.resource_type,
resource_type: resource.resource_type as 'zim' | 'map',
installed_version: resource.version,
latest_version: latest.version,
download_url: latest.download_url,

24
admin/app/services/content_auto_update_service.ts
View File

@ -269,9 +269,15 @@ export class ContentAutoUpdateService {
await this.maybeResetWindowBudget(config, now)
// Local catalog check + persist available-update state for every resource.
const installed = await InstalledResource.all()
// ZIM/map catalog path only — `dataset` resources are excluded (they get
// their own freshness path; no-op today as none are written in this slice).
const installed = await InstalledResource.query().whereNot('resource_type', 'dataset')
const latestByKey = await this.catalog.getLatestForResources(
installed.map((r) => ({ resource_id: r.resource_id, resource_type: r.resource_type }))
// `dataset` rows are filtered out above, so the type narrows to ZIM/map.
installed.map((r) => ({
resource_id: r.resource_id,
resource_type: r.resource_type as 'zim' | 'map',
}))
)
for (const resource of installed) {
const latest = latestByKey.get(`${resource.resource_type}:${resource.resource_id}`) ?? null
@ -321,7 +327,8 @@ export class ContentAutoUpdateService {
const result = await this.collectionUpdateService.applyUpdate(
{
resource_id: candidate.resource.resource_id,
resource_type: candidate.resource.resource_type,
// `dataset` rows are filtered out of `installed` above, so ZIM/map.
resource_type: candidate.resource.resource_type as 'zim' | 'map',
installed_version: candidate.resource.version,
latest_version: candidate.version,
download_url: candidate.download_url,
@ -407,7 +414,9 @@ export class ContentAutoUpdateService {
const now = overrides.now ?? DateTime.now()
const withinWindow = isWithinWindow(config.windowStart, config.windowEnd, now)
const pending = await InstalledResource.query().whereNotNull('available_update_version')
const pending = await InstalledResource.query()
.whereNotNull('available_update_version')
.whereNot('resource_type', 'dataset')
const eligible = pending.filter(
(r) => this.resourceEligibility(r, config.cooloffHours, now).eligible
)
@ -506,7 +515,9 @@ export class ContentAutoUpdateService {
const config = await this.getConfig()
const now = DateTime.now()
const pending = await InstalledResource.query().whereNotNull('available_update_version')
const pending = await InstalledResource.query()
.whereNotNull('available_update_version')
.whereNot('resource_type', 'dataset')
const resources: ContentAutoUpdateResourceStatus[] = pending.map((resource) => {
const verdict = this.resourceEligibility(resource, config.cooloffHours, now)
const size = resource.available_update_size_bytes ?? null
@ -514,7 +525,8 @@ export class ContentAutoUpdateService {
config.maxBytesPerWindow > 0 && size !== null && size > config.maxBytesPerWindow
return {
resource_id: resource.resource_id,
resource_type: resource.resource_type,
// `dataset` rows are filtered out of `pending` above, so ZIM/map.
resource_type: resource.resource_type as 'zim' | 'map',
current_version: resource.version,
available_update_version: resource.available_update_version,
size_bytes: size,

26
admin/app/services/zim_service.ts
View File

@ -27,6 +27,8 @@ import WikipediaSelection from '#models/wikipedia_selection'
import InstalledResource from '#models/installed_resource'
import CollectionManifest from '#models/collection_manifest'
import { RunDownloadJob } from '#jobs/run_download_job'
import { DownloadDrugDataJob } from '#jobs/download_drug_data_job'
import { DrugReferenceService } from './drug_reference_service.js'
import { SERVICE_NAMES } from '../../constants/service_names.js'
import { CollectionManifestService } from './collection_manifest_service.js'
import { KiwixLibraryService } from './kiwix_library_service.js'
@ -263,6 +265,30 @@ export class ZimService {
const downloadFilenames: string[] = []
for (const resource of toDownload) {
// A `dataset` resource (e.g. the FDA drug labels) is DB-ingested, not a
// ZIM file — route it to the drug download+ingest pipeline instead of
// RunDownloadJob. The installed-filter above queries resource_type='zim',
// so a dataset is never seen as installed and would re-dispatch the ~1.7 GB
// download on every tier select. Guard against that with the existing
// drug-status signal: skip when the data is already present.
// (Planned follow-up: write an InstalledResource 'dataset' row on ingest
// 'ready' so the tier-status/badge math recognizes it uniformly — a
// maintainer decision still pending, out of scope for this slice.)
if (resource.type === 'dataset') {
const drugReferenceService = new DrugReferenceService()
const status = await drugReferenceService.getIngestStatus()
if (status.phase === 'ready' || status.rowCount > 0) {
logger.info('[ZimService] Drug dataset already ingested, skipping dispatch.')
continue
}
// DownloadDrugDataJob.dispatch() is idempotent on its deterministic
// jobId — a concurrent in-flight download returns "already running"
// without re-adding, so this is safe to call repeatedly.
await DownloadDrugDataJob.dispatch()
logger.info('[ZimService] Dispatched drug data download for dataset resource.')
continue
}
const existingJob = await RunDownloadJob.getActiveByUrl(resource.url)
if (existingJob) {
logger.warn(`[ZimService] Download already in progress for ${resource.url}, skipping.`)

3
admin/app/validators/curated_collections.ts
View File

@ -9,6 +9,9 @@ export const specResourceValidator = vine.object({
description: vine.string(),
url: vine.string().url(),
size_mb: vine.number().min(0).optional(),
// Resource-type discriminator (absent == 'zim'). Required here because VineJS
// strips unknown keys, which would silently drop the field on manifest fetch.
type: vine.enum(['zim', 'dataset']).optional(),
})
// ---- ZIM Categories spec (versioned) ----

6
admin/types/collections.ts
View File

@ -5,6 +5,12 @@ export type SpecResource = {
description: string
url: string
size_mb: number
/**
* Resource-type discriminator. Absent == 'zim' so every existing manifest
* entry keeps the current ZIM download path. 'dataset' routes the tier
* installer to the DB-ingested drug pipeline instead.
*/
type?: 'zim' | 'dataset'
}
export type SpecTier = {