fix(downloads): let interrupted content downloads resume (#1202)

doResumableDownload already implements resume - stats the .tmp, sends a
Range header, handles a server that ignores it. The code was unreachable
for ZIMs because every dispatch site passed forceNew: true, which skips
the partial-file check and opens the stream with 'w' instead of 'a', so
an interrupted 12.5 GB Wikipedia download truncated and restarted at
byte 0. Maps already do this correctly (map_service.ts:681, "so retries
resume partial downloads"); ZIMs never got the same treatment.

Worse in combination with attempts: 10 - every retry also restarted from
zero, so a flaky connection re-downloaded the whole file up to ten times.

Drop forceNew from the content-download dispatch sites (it already
defaults to false) and add the guard that enabling resume requires: a
.tmp larger than the file now on the server cannot be a prefix of it,
because openZIM re-publishes builds under the same name. Resuming would
request a range past the end, 416 on every attempt, and never delete the
.tmp - so the download could never recover on its own.

Verified on a test appliance: a 470 MB partial survived a container
restart and continued rather than truncating, and a planted oversized
.tmp was discarded with the file then downloading to its exact size.

Refs #1201
This commit is contained in:
chriscrosstalk 2026-08-03 19:56:50 -07:00 committed by jakeaturner
parent 7472442ae8
commit 2159c9dec6
No known key found for this signature in database
GPG Key ID: B1072EBDEECE328D
4 changed files with 15 additions and 5 deletions

View File

@ -120,7 +120,6 @@ export class CollectionUpdateService {
timeout: 30000,
allowedMimeTypes:
update.resource_type === 'zim' ? ZIM_MIME_TYPES : PMTILES_MIME_TYPES,
forceNew: true,
filetype: update.resource_type,
title: update.resource_id,
totalBytes: update.size_bytes,

View File

@ -117,7 +117,6 @@ export class CreatorPackService {
filepath,
timeout: 30000,
allowedMimeTypes: ZIM_MIME_TYPES,
forceNew: true,
filetype: 'zim',
title: pack.name,
totalBytes: pack.size_mb ? pack.size_mb * 1024 * 1024 : undefined,

View File

@ -222,7 +222,6 @@ export class ZimService {
filepath,
timeout: 30000,
allowedMimeTypes: ZIM_MIME_TYPES,
forceNew: true,
filetype: 'zim',
title: metadata?.title,
totalBytes: metadata?.size_bytes,
@ -331,7 +330,6 @@ export class ZimService {
filepath,
timeout: 30000,
allowedMimeTypes: ZIM_MIME_TYPES,
forceNew: true,
filetype: 'zim',
title: (resource as any).title || undefined,
totalBytes: resolved.sizeBytes,
@ -830,7 +828,6 @@ export class ZimService {
filepath,
timeout: 30000,
allowedMimeTypes: ZIM_MIME_TYPES,
forceNew: true,
filetype: 'zim',
title: selectedOption.name,
totalBytes: selectedOption.size_mb ? selectedOption.size_mb * 1024 * 1024 : undefined,

View File

@ -8,6 +8,7 @@ import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from
import { createWriteStream } from 'fs'
import { rename } from 'fs/promises'
import path from 'path'
import logger from '@adonisjs/core/services/logger'
// Some upstream mirrors reject requests with a missing or generic User-Agent.
// Notably, download.kiwix.org routes the large Wikimedia-family ZIMs (Wikipedia,
@ -121,6 +122,20 @@ export async function doResumableDownload({
appendMode = false
}
// A .tmp bigger than the file now on the server cannot be a prefix of it — the
// publisher replaced the file under the same name (openZIM rolls builds forward,
// see #1189/#1187). Resuming would ask for a range past the end and get a 416 on
// every attempt, with nothing deleting the .tmp, so the download could never
// recover on its own. Discard and start clean.
if (startByte > totalBytes && totalBytes > 0) {
logger.warn(
`[Download] Discarding stale partial for ${filepath}: .tmp is ${startByte}B but the server reports ${totalBytes}B`
)
await deleteFileIfExists(tempPath)
startByte = 0
appendMode = false
}
// Add Range header if resuming
if (supportsRangeRequests && startByte > 0) {
headers.Range = `bytes=${startByte}-`