From 223cde5774410cc3057d860ef8fdda4a4f095628 Mon Sep 17 00:00:00 2001 From: NgoQuocViet2001 Date: Mon, 13 Jul 2026 14:07:42 +0700 Subject: [PATCH] fix(content): resolve current ZIM URL before download --- admin/app/services/zim_service.ts | 21 +++++-- admin/app/utils/zim_download_resolution.ts | 29 ++++++++++ .../unit/zim_download_resolution.spec.ts | 56 +++++++++++++++++++ 3 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 admin/app/utils/zim_download_resolution.ts create mode 100644 admin/tests/unit/zim_download_resolution.spec.ts diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 5a362e0..3a411fa 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -29,10 +29,12 @@ import CollectionManifest from '#models/collection_manifest' import { RunDownloadJob } from '#jobs/run_download_job' import { SERVICE_NAMES } from '../../constants/service_names.js' import { CollectionManifestService } from './collection_manifest_service.js' +import { KiwixCatalogService } from './kiwix_catalog_service.js' import { KiwixLibraryService } from './kiwix_library_service.js' import type { CategoryWithStatus } from '../../types/collections.js' import CustomLibrarySource from '#models/custom_library_source' import { assertNotPrivateUrl } from '#validators/common' +import { resolveZimDownload } from '../utils/zim_download_resolution.js' const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream'] const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json' @@ -260,33 +262,40 @@ export class ZimService { if (toDownload.length === 0) return null + const latestByResource = await new KiwixCatalogService().getLatestForResources( + toDownload.map((resource) => ({ resource_id: resource.id, resource_type: 'zim' })) + ) const downloadFilenames: string[] = [] for (const resource of toDownload) { - const existingJob = await RunDownloadJob.getActiveByUrl(resource.url) + const resolved = resolveZimDownload( + resource, + latestByResource.get(`zim:${resource.id}`) ?? null + ) + const existingJob = await RunDownloadJob.getActiveByUrl(resolved.url) if (existingJob) { - logger.warn(`[ZimService] Download already in progress for ${resource.url}, skipping.`) + logger.warn(`[ZimService] Download already in progress for ${resolved.url}, skipping.`) continue } - const filename = resource.url.split('/').pop() + const filename = resolved.url.split('/').pop() if (!filename) continue downloadFilenames.push(filename) const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename) await RunDownloadJob.dispatch({ - url: resource.url, + url: resolved.url, filepath, timeout: 30000, allowedMimeTypes: ZIM_MIME_TYPES, forceNew: true, filetype: 'zim', title: (resource as any).title || undefined, - totalBytes: (resource as any).size_mb ? (resource as any).size_mb * 1024 * 1024 : undefined, + totalBytes: resolved.sizeBytes, resourceMetadata: { resource_id: resource.id, - version: resource.version, + version: resolved.version, collection_ref: categorySlug, }, }) diff --git a/admin/app/utils/zim_download_resolution.ts b/admin/app/utils/zim_download_resolution.ts new file mode 100644 index 0000000..d78051f --- /dev/null +++ b/admin/app/utils/zim_download_resolution.ts @@ -0,0 +1,29 @@ +import type { CatalogResult } from '../services/kiwix_catalog_service.js' +import type { SpecResource } from '../../types/collections.js' + +export type ResolvedZimDownload = { + url: string + version: string + sizeBytes: number | undefined +} + +export function resolveZimDownload( + resource: SpecResource, + latest: CatalogResult | null +): ResolvedZimDownload { + const manifestSizeBytes = resource.size_mb > 0 ? resource.size_mb * 1024 * 1024 : undefined + + if (!latest || latest.version < resource.version) { + return { + url: resource.url, + version: resource.version, + sizeBytes: manifestSizeBytes, + } + } + + return { + url: latest.download_url, + version: latest.version, + sizeBytes: latest.size_bytes > 0 ? latest.size_bytes : manifestSizeBytes, + } +} diff --git a/admin/tests/unit/zim_download_resolution.spec.ts b/admin/tests/unit/zim_download_resolution.spec.ts new file mode 100644 index 0000000..cc076b7 --- /dev/null +++ b/admin/tests/unit/zim_download_resolution.spec.ts @@ -0,0 +1,56 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { resolveZimDownload } from '../../app/utils/zim_download_resolution.js' + +const manifestResource = { + id: 'wikipedia_en_all_mini', + version: '2025-12', + title: 'Wikipedia', + description: 'Compact Wikipedia', + url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2025-12.zim', + size_mb: 11_400, +} + +test('live catalog result replaces stale manifest download metadata', () => { + const resolved = resolveZimDownload(manifestResource, { + version: '2026-06', + download_url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim', + size_bytes: 12_531_944_448, + }) + + assert.deepEqual(resolved, { + url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim', + version: '2026-06', + sizeBytes: 12_531_944_448, + }) +}) + +test('missing catalog result falls back to static manifest metadata', () => { + assert.deepEqual(resolveZimDownload(manifestResource, null), { + url: manifestResource.url, + version: manifestResource.version, + sizeBytes: manifestResource.size_mb * 1024 * 1024, + }) +}) + +test('catalog result with unknown size keeps the manifest size estimate', () => { + const resolved = resolveZimDownload(manifestResource, { + version: '2026-06', + download_url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim', + size_bytes: 0, + }) + + assert.equal(resolved.sizeBytes, manifestResource.size_mb * 1024 * 1024) +}) + +test('older catalog result does not replace newer manifest metadata', () => { + const resolved = resolveZimDownload(manifestResource, { + version: '2025-09', + download_url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2025-09.zim', + size_bytes: 10_000, + }) + + assert.equal(resolved.url, manifestResource.url) + assert.equal(resolved.version, manifestResource.version) +})