Merge 223cde5774 into 6a4f02dd46

admin/app/services/zim_service.ts

@@ -29,10 +29,12 @@ import CollectionManifest from '#models/collection_manifest'
 import { RunDownloadJob } from '#jobs/run_download_job'
 import { SERVICE_NAMES } from '../../constants/service_names.js'
 import { CollectionManifestService } from './collection_manifest_service.js'
+import { KiwixCatalogService } from './kiwix_catalog_service.js'
 import { KiwixLibraryService } from './kiwix_library_service.js'
 import type { CategoryWithStatus } from '../../types/collections.js'
 import CustomLibrarySource from '#models/custom_library_source'
 import { assertNotPrivateUrl } from '#validators/common'
+import { resolveZimDownload } from '../utils/zim_download_resolution.js'
 
 const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
 const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json'
@@ -260,33 +262,40 @@ export class ZimService {
 
     if (toDownload.length === 0) return null
 
+    const latestByResource = await new KiwixCatalogService().getLatestForResources(
+      toDownload.map((resource) => ({ resource_id: resource.id, resource_type: 'zim' }))
+    )
     const downloadFilenames: string[] = []
 
     for (const resource of toDownload) {
-      const existingJob = await RunDownloadJob.getActiveByUrl(resource.url)
+      const resolved = resolveZimDownload(
+        resource,
+        latestByResource.get(`zim:${resource.id}`) ?? null
+      )
+      const existingJob = await RunDownloadJob.getActiveByUrl(resolved.url)
       if (existingJob) {
-        logger.warn(`[ZimService] Download already in progress for ${resource.url}, skipping.`)
+        logger.warn(`[ZimService] Download already in progress for ${resolved.url}, skipping.`)
         continue
       }
 
-      const filename = resource.url.split('/').pop()
+      const filename = resolved.url.split('/').pop()
       if (!filename) continue
 
       downloadFilenames.push(filename)
       const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename)
 
       await RunDownloadJob.dispatch({
-        url: resource.url,
+        url: resolved.url,
         filepath,
         timeout: 30000,
         allowedMimeTypes: ZIM_MIME_TYPES,
         forceNew: true,
         filetype: 'zim',
         title: (resource as any).title || undefined,
-        totalBytes: (resource as any).size_mb ? (resource as any).size_mb * 1024 * 1024 : undefined,
+        totalBytes: resolved.sizeBytes,
         resourceMetadata: {
           resource_id: resource.id,
-          version: resource.version,
+          version: resolved.version,
           collection_ref: categorySlug,
         },
       })

admin/app/utils/zim_download_resolution.ts

@@ -0,0 +1,29 @@
+import type { CatalogResult } from '../services/kiwix_catalog_service.js'
+import type { SpecResource } from '../../types/collections.js'
+
+export type ResolvedZimDownload = {
+  url: string
+  version: string
+  sizeBytes: number | undefined
+}
+
+export function resolveZimDownload(
+  resource: SpecResource,
+  latest: CatalogResult | null
+): ResolvedZimDownload {
+  const manifestSizeBytes = resource.size_mb > 0 ? resource.size_mb * 1024 * 1024 : undefined
+
+  if (!latest || latest.version < resource.version) {
+    return {
+      url: resource.url,
+      version: resource.version,
+      sizeBytes: manifestSizeBytes,
+    }
+  }
+
+  return {
+    url: latest.download_url,
+    version: latest.version,
+    sizeBytes: latest.size_bytes > 0 ? latest.size_bytes : manifestSizeBytes,
+  }
+}

admin/tests/unit/zim_download_resolution.spec.ts

@@ -0,0 +1,56 @@
+import * as assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import { resolveZimDownload } from '../../app/utils/zim_download_resolution.js'
+
+const manifestResource = {
+  id: 'wikipedia_en_all_mini',
+  version: '2025-12',
+  title: 'Wikipedia',
+  description: 'Compact Wikipedia',
+  url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2025-12.zim',
+  size_mb: 11_400,
+}
+
+test('live catalog result replaces stale manifest download metadata', () => {
+  const resolved = resolveZimDownload(manifestResource, {
+    version: '2026-06',
+    download_url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim',
+    size_bytes: 12_531_944_448,
+  })
+
+  assert.deepEqual(resolved, {
+    url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim',
+    version: '2026-06',
+    sizeBytes: 12_531_944_448,
+  })
+})
+
+test('missing catalog result falls back to static manifest metadata', () => {
+  assert.deepEqual(resolveZimDownload(manifestResource, null), {
+    url: manifestResource.url,
+    version: manifestResource.version,
+    sizeBytes: manifestResource.size_mb * 1024 * 1024,
+  })
+})
+
+test('catalog result with unknown size keeps the manifest size estimate', () => {
+  const resolved = resolveZimDownload(manifestResource, {
+    version: '2026-06',
+    download_url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2026-06.zim',
+    size_bytes: 0,
+  })
+
+  assert.equal(resolved.sizeBytes, manifestResource.size_mb * 1024 * 1024)
+})
+
+test('older catalog result does not replace newer manifest metadata', () => {
+  const resolved = resolveZimDownload(manifestResource, {
+    version: '2025-09',
+    download_url: 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2025-09.zim',
+    size_bytes: 10_000,
+  })
+
+  assert.equal(resolved.url, manifestResource.url)
+  assert.equal(resolved.version, manifestResource.version)
+})