fix(downloads): let interrupted content downloads resume (#1202)
doResumableDownload already implements resume - stats the .tmp, sends a Range header, handles a server that ignores it. The code was unreachable for ZIMs because every dispatch site passed forceNew: true, which skips the partial-file check and opens the stream with 'w' instead of 'a', so an interrupted 12.5 GB Wikipedia download truncated and restarted at byte 0. Maps already do this correctly (map_service.ts:681, "so retries resume partial downloads"); ZIMs never got the same treatment. Worse in combination with attempts: 10 - every retry also restarted from zero, so a flaky connection re-downloaded the whole file up to ten times. Drop forceNew from the content-download dispatch sites (it already defaults to false) and add the guard that enabling resume requires: a .tmp larger than the file now on the server cannot be a prefix of it, because openZIM re-publishes builds under the same name. Resuming would request a range past the end, 416 on every attempt, and never delete the .tmp - so the download could never recover on its own. Verified on a test appliance: a 470 MB partial survived a container restart and continued rather than truncating, and a planted oversized .tmp was discarded with the file then downloading to its exact size. Refs #1201
This commit is contained in:
parent
7472442ae8
commit
2159c9dec6
|
|
@ -120,7 +120,6 @@ export class CollectionUpdateService {
|
|||
timeout: 30000,
|
||||
allowedMimeTypes:
|
||||
update.resource_type === 'zim' ? ZIM_MIME_TYPES : PMTILES_MIME_TYPES,
|
||||
forceNew: true,
|
||||
filetype: update.resource_type,
|
||||
title: update.resource_id,
|
||||
totalBytes: update.size_bytes,
|
||||
|
|
|
|||
|
|
@ -117,7 +117,6 @@ export class CreatorPackService {
|
|||
filepath,
|
||||
timeout: 30000,
|
||||
allowedMimeTypes: ZIM_MIME_TYPES,
|
||||
forceNew: true,
|
||||
filetype: 'zim',
|
||||
title: pack.name,
|
||||
totalBytes: pack.size_mb ? pack.size_mb * 1024 * 1024 : undefined,
|
||||
|
|
|
|||
|
|
@ -222,7 +222,6 @@ export class ZimService {
|
|||
filepath,
|
||||
timeout: 30000,
|
||||
allowedMimeTypes: ZIM_MIME_TYPES,
|
||||
forceNew: true,
|
||||
filetype: 'zim',
|
||||
title: metadata?.title,
|
||||
totalBytes: metadata?.size_bytes,
|
||||
|
|
@ -331,7 +330,6 @@ export class ZimService {
|
|||
filepath,
|
||||
timeout: 30000,
|
||||
allowedMimeTypes: ZIM_MIME_TYPES,
|
||||
forceNew: true,
|
||||
filetype: 'zim',
|
||||
title: (resource as any).title || undefined,
|
||||
totalBytes: resolved.sizeBytes,
|
||||
|
|
@ -830,7 +828,6 @@ export class ZimService {
|
|||
filepath,
|
||||
timeout: 30000,
|
||||
allowedMimeTypes: ZIM_MIME_TYPES,
|
||||
forceNew: true,
|
||||
filetype: 'zim',
|
||||
title: selectedOption.name,
|
||||
totalBytes: selectedOption.size_mb ? selectedOption.size_mb * 1024 * 1024 : undefined,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from
|
|||
import { createWriteStream } from 'fs'
|
||||
import { rename } from 'fs/promises'
|
||||
import path from 'path'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
// Some upstream mirrors reject requests with a missing or generic User-Agent.
|
||||
// Notably, download.kiwix.org routes the large Wikimedia-family ZIMs (Wikipedia,
|
||||
|
|
@ -121,6 +122,20 @@ export async function doResumableDownload({
|
|||
appendMode = false
|
||||
}
|
||||
|
||||
// A .tmp bigger than the file now on the server cannot be a prefix of it — the
|
||||
// publisher replaced the file under the same name (openZIM rolls builds forward,
|
||||
// see #1189/#1187). Resuming would ask for a range past the end and get a 416 on
|
||||
// every attempt, with nothing deleting the .tmp, so the download could never
|
||||
// recover on its own. Discard and start clean.
|
||||
if (startByte > totalBytes && totalBytes > 0) {
|
||||
logger.warn(
|
||||
`[Download] Discarding stale partial for ${filepath}: .tmp is ${startByte}B but the server reports ${totalBytes}B`
|
||||
)
|
||||
await deleteFileIfExists(tempPath)
|
||||
startByte = 0
|
||||
appendMode = false
|
||||
}
|
||||
|
||||
// Add Range header if resuming
|
||||
if (supportsRangeRequests && startByte > 0) {
|
||||
headers.Range = `bytes=${startByte}-`
|
||||
|
|
|
|||
Loading…
Reference in New Issue