fix(content): resolve current ZIM URL before download

This commit is contained in:
NgoQuocViet2001 2026-07-13 14:07:42 +07:00
parent 6a4f02dd46
commit 223cde5774
3 changed files with 100 additions and 6 deletions

View File

@ -29,10 +29,12 @@ import CollectionManifest from '#models/collection_manifest'
import { RunDownloadJob } from '#jobs/run_download_job'
import { SERVICE_NAMES } from '../../constants/service_names.js'
import { CollectionManifestService } from './collection_manifest_service.js'
import { KiwixCatalogService } from './kiwix_catalog_service.js'
import { KiwixLibraryService } from './kiwix_library_service.js'
import type { CategoryWithStatus } from '../../types/collections.js'
import CustomLibrarySource from '#models/custom_library_source'
import { assertNotPrivateUrl } from '#validators/common'
import { resolveZimDownload } from '../utils/zim_download_resolution.js'
const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json'
@ -260,33 +262,40 @@ export class ZimService {
if (toDownload.length === 0) return null
const latestByResource = await new KiwixCatalogService().getLatestForResources(
toDownload.map((resource) => ({ resource_id: resource.id, resource_type: 'zim' }))
)
const downloadFilenames: string[] = []
for (const resource of toDownload) {
const existingJob = await RunDownloadJob.getActiveByUrl(resource.url)
const resolved = resolveZimDownload(
resource,
latestByResource.get(`zim:${resource.id}`) ?? null
)
const existingJob = await RunDownloadJob.getActiveByUrl(resolved.url)
if (existingJob) {
logger.warn(`[ZimService] Download already in progress for ${resource.url}, skipping.`)
logger.warn(`[ZimService] Download already in progress for ${resolved.url}, skipping.`)
continue
}
const filename = resource.url.split('/').pop()
const filename = resolved.url.split('/').pop()
if (!filename) continue
downloadFilenames.push(filename)
const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename)
await RunDownloadJob.dispatch({
url: resource.url,
url: resolved.url,
filepath,
timeout: 30000,
allowedMimeTypes: ZIM_MIME_TYPES,
forceNew: true,
filetype: 'zim',
title: (resource as any).title || undefined,
totalBytes: (resource as any).size_mb ? (resource as any).size_mb * 1024 * 1024 : undefined,
totalBytes: resolved.sizeBytes,
resourceMetadata: {
resource_id: resource.id,
version: resource.version,
version: resolved.version,
collection_ref: categorySlug,
},
})

View File

@ -0,0 +1,29 @@
import type { CatalogResult } from '../services/kiwix_catalog_service.js'
import type { SpecResource } from '../../types/collections.js'
export type ResolvedZimDownload = {
url: string
version: string
sizeBytes: number | undefined
}
export function resolveZimDownload(
resource: SpecResource,
latest: CatalogResult | null
): ResolvedZimDownload {
const manifestSizeBytes = resource.size_mb > 0 ? resource.size_mb * 1024 * 1024 : undefined
if (!latest || latest.version < resource.version) {
return {
url: resource.url,
version: resource.version,
sizeBytes: manifestSizeBytes,
}
}
return {
url: latest.download_url,
version: latest.version,
sizeBytes: latest.size_bytes > 0 ? latest.size_bytes : manifestSizeBytes,
}
}

View File

@ -0,0 +1,56 @@
import * as assert from 'node:assert/strict'
import { test } from 'node:test'
import { resolveZimDownload } from '../../app/utils/zim_download_resolution.js'
const manifestResource = {
id: 'wikipedia_en_all_mini',
version: '2025-12',
title: 'Wikipedia',
description: 'Compact Wikipedia',
url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2025-12.zim',
size_mb: 11_400,
}
test('live catalog result replaces stale manifest download metadata', () => {
const resolved = resolveZimDownload(manifestResource, {
version: '2026-06',
download_url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim',
size_bytes: 12_531_944_448,
})
assert.deepEqual(resolved, {
url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim',
version: '2026-06',
sizeBytes: 12_531_944_448,
})
})
test('missing catalog result falls back to static manifest metadata', () => {
assert.deepEqual(resolveZimDownload(manifestResource, null), {
url: manifestResource.url,
version: manifestResource.version,
sizeBytes: manifestResource.size_mb * 1024 * 1024,
})
})
test('catalog result with unknown size keeps the manifest size estimate', () => {
const resolved = resolveZimDownload(manifestResource, {
version: '2026-06',
download_url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim',
size_bytes: 0,
})
assert.equal(resolved.sizeBytes, manifestResource.size_mb * 1024 * 1024)
})
test('older catalog result does not replace newer manifest metadata', () => {
const resolved = resolveZimDownload(manifestResource, {
version: '2025-09',
download_url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2025-09.zim',
size_bytes: 10_000,
})
assert.equal(resolved.url, manifestResource.url)
assert.equal(resolved.version, manifestResource.version)
})