diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index c305939..bce5d76 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -29,10 +29,12 @@ import CollectionManifest from '#models/collection_manifest' import { RunDownloadJob } from '#jobs/run_download_job' import { SERVICE_NAMES } from '../../constants/service_names.js' import { CollectionManifestService } from './collection_manifest_service.js' +import { KiwixCatalogService } from './kiwix_catalog_service.js' import { KiwixLibraryService } from './kiwix_library_service.js' import type { CategoryWithStatus } from '../../types/collections.js' import CustomLibrarySource from '#models/custom_library_source' import { assertNotPrivateUrl } from '#validators/common' +import { resolveZimDownload } from '../utils/zim_download_resolution.js' const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream'] const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json' @@ -267,33 +269,40 @@ export class ZimService { if (toDownload.length === 0) return null + const latestByResource = await new KiwixCatalogService().getLatestForResources( + toDownload.map((resource) => ({ resource_id: resource.id, resource_type: 'zim' })) + ) const downloadFilenames: string[] = [] for (const resource of toDownload) { - const existingJob = await RunDownloadJob.getActiveByUrl(resource.url) + const resolved = resolveZimDownload( + resource, + latestByResource.get(`zim:${resource.id}`) ?? null + ) + const existingJob = await RunDownloadJob.getActiveByUrl(resolved.url) if (existingJob) { - logger.warn(`[ZimService] Download already in progress for ${resource.url}, skipping.`) + logger.warn(`[ZimService] Download already in progress for ${resolved.url}, skipping.`) continue } - const filename = resource.url.split('/').pop() + const filename = resolved.url.split('/').pop() if (!filename) continue downloadFilenames.push(filename) const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename) await RunDownloadJob.dispatch({ - url: resource.url, + url: resolved.url, filepath, timeout: 30000, allowedMimeTypes: ZIM_MIME_TYPES, forceNew: true, filetype: 'zim', title: (resource as any).title || undefined, - totalBytes: (resource as any).size_mb ? (resource as any).size_mb * 1024 * 1024 : undefined, + totalBytes: resolved.sizeBytes, resourceMetadata: { resource_id: resource.id, - version: resource.version, + version: resolved.version, collection_ref: categorySlug, }, }) diff --git a/admin/app/utils/zim_download_resolution.ts b/admin/app/utils/zim_download_resolution.ts new file mode 100644 index 0000000..9817453 --- /dev/null +++ b/admin/app/utils/zim_download_resolution.ts @@ -0,0 +1,47 @@ +import type { CatalogResult } from '../services/kiwix_catalog_service.js' +import type { SpecResource } from '../../types/collections.js' + +export type ResolvedZimDownload = { + url: string + version: string + sizeBytes: number | undefined +} + +function compareZimVersions(left: string, right: string): number { + const parse = (value: string): [number, number] | null => { + const match = /^(\d{4})-(\d{1,2})$/.exec(value) + if (!match) return null + + const month = Number.parseInt(match[2], 10) + if (month < 1 || month > 12) return null + + return [Number.parseInt(match[1], 10), month] + } + + const leftParts = parse(left) + const rightParts = parse(right) + if (!leftParts || !rightParts) return left.localeCompare(right) + + return leftParts[0] - rightParts[0] || leftParts[1] - rightParts[1] +} + +export function resolveZimDownload( + resource: SpecResource, + latest: CatalogResult | null +): ResolvedZimDownload { + const manifestSizeBytes = resource.size_mb > 0 ? resource.size_mb * 1024 * 1024 : undefined + + if (!latest || compareZimVersions(latest.version, resource.version) < 0) { + return { + url: resource.url, + version: resource.version, + sizeBytes: manifestSizeBytes, + } + } + + return { + url: latest.download_url, + version: latest.version, + sizeBytes: latest.size_bytes > 0 ? latest.size_bytes : manifestSizeBytes, + } +} diff --git a/admin/tests/unit/zim_download_resolution.spec.ts b/admin/tests/unit/zim_download_resolution.spec.ts new file mode 100644 index 0000000..11f55ce --- /dev/null +++ b/admin/tests/unit/zim_download_resolution.spec.ts @@ -0,0 +1,75 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { resolveZimDownload } from '../../app/utils/zim_download_resolution.js' + +const manifestResource = { + id: 'wikipedia_en_all_mini', + version: '2025-12', + title: 'Wikipedia', + description: 'Compact Wikipedia', + url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2025-12.zim', + size_mb: 11_400, +} + +test('live catalog result replaces stale manifest download metadata', () => { + const resolved = resolveZimDownload(manifestResource, { + version: '2026-06', + download_url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim', + size_bytes: 12_531_944_448, + }) + + assert.deepEqual(resolved, { + url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim', + version: '2026-06', + sizeBytes: 12_531_944_448, + }) +}) + +test('missing catalog result falls back to static manifest metadata', () => { + assert.deepEqual(resolveZimDownload(manifestResource, null), { + url: manifestResource.url, + version: manifestResource.version, + sizeBytes: manifestResource.size_mb * 1024 * 1024, + }) +}) + +test('catalog result with unknown size keeps the manifest size estimate', () => { + const resolved = resolveZimDownload(manifestResource, { + version: '2026-06', + download_url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim', + size_bytes: 0, + }) + + assert.equal(resolved.sizeBytes, manifestResource.size_mb * 1024 * 1024) +}) + +test('older catalog result does not replace newer manifest metadata', () => { + const resolved = resolveZimDownload(manifestResource, { + version: '2025-09', + download_url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2025-09.zim', + size_bytes: 10_000, + }) + + assert.equal(resolved.url, manifestResource.url) + assert.equal(resolved.version, manifestResource.version) +}) + +test('non-padded catalog months are compared numerically', () => { + const resource = { + ...manifestResource, + version: '2026-2', + url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-2.zim', + } + const resolved = resolveZimDownload(resource, { + version: '2026-10', + download_url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-10.zim', + size_bytes: 13_000, + }) + + assert.equal( + resolved.url, + 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-10.zim' + ) + assert.equal(resolved.version, '2026-10') +})