From e214bf8cc31c11a18dbfe54d3fa9200215e92389 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Thu, 16 Jul 2026 15:19:44 -0700 Subject: [PATCH] fix(downloads): send a descriptive User-Agent so Wikimedia mirrors don't 403 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit download.kiwix.org routes the large Wikimedia-family ZIMs (Wikipedia, Wikiversity, Wikibooks — including the flagship full Wikipedia) to dumps.wikimedia.org, which enforces a User-Agent policy and returns HTTP 403 for requests with a missing or generic (axios/x) User-Agent. Because doResumableDownload sent no User-Agent, every Wikimedia-hosted ZIM failed to download while Kiwix-mirror-hosted ZIMs succeeded — so a curated set or Easy Setup run would silently stall on exactly the highest-value content. Add a descriptive User-Agent to the HEAD and GET requests. Verified against the live mirror: default/empty UA -> 403, ProjectNOMAD UA -> 200/206. Co-Authored-By: Claude Opus 4.8 (1M context) --- admin/app/utils/downloads.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/admin/app/utils/downloads.ts b/admin/app/utils/downloads.ts index bfe189a..38213a4 100644 --- a/admin/app/utils/downloads.ts +++ b/admin/app/utils/downloads.ts @@ -9,6 +9,16 @@ import { createWriteStream } from 'fs' import { rename } from 'fs/promises' import path from 'path' +// Some upstream mirrors reject requests with a missing or generic User-Agent. +// Notably, download.kiwix.org routes the large Wikimedia-family ZIMs (Wikipedia, +// Wikiversity, Wikibooks — including the flagship full Wikipedia) to +// dumps.wikimedia.org, which returns HTTP 403 for a default `axios/x` (or empty) +// User-Agent per Wikimedia's UA policy. Identify ourselves descriptively so +// those downloads succeed. +const DOWNLOAD_HEADERS: Record = { + 'User-Agent': 'ProjectNOMAD/1.0 (+https://projectnomad.us)', +} + /** * Perform a resumable download with progress tracking * @param param0 - Download parameters. Leave allowedMimeTypes empty to skip mime type checking. @@ -45,6 +55,7 @@ export async function doResumableDownload({ const headResponse = await axios.head(url, { signal, timeout, + headers: DOWNLOAD_HEADERS, }) // Some upstream hosts (notably download.kiwix.org for .zim files) don't set a @@ -94,7 +105,12 @@ export async function doResumableDownload({ } const fetchStream = (hdrs: Record) => - axios.get(url, { responseType: 'stream', headers: hdrs, signal, timeout }) + axios.get(url, { + responseType: 'stream', + headers: { ...DOWNLOAD_HEADERS, ...hdrs }, + signal, + timeout, + }) let response = await fetchStream(headers)