fix(downloads): send a descriptive User-Agent so Wikimedia mirrors don't 403 (#1114)

download.kiwix.org routes the large Wikimedia-family ZIMs (Wikipedia,
Wikiversity, Wikibooks — including the flagship full Wikipedia) to
dumps.wikimedia.org, which enforces a User-Agent policy and returns HTTP
403 for requests with a missing or generic (axios/x) User-Agent. Because
doResumableDownload sent no User-Agent, every Wikimedia-hosted ZIM failed
to download while Kiwix-mirror-hosted ZIMs succeeded — so a curated set or
Easy Setup run would silently stall on exactly the highest-value content.

Add a descriptive User-Agent to the HEAD and GET requests. Verified
against the live mirror: default/empty UA -> 403, ProjectNOMAD UA -> 200/206.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

admin/app/utils/downloads.ts

@@ -9,6 +9,16 @@ import { createWriteStream } from 'fs'
 import { rename } from 'fs/promises'
 import path from 'path'
 
+// Some upstream mirrors reject requests with a missing or generic User-Agent.
+// Notably, download.kiwix.org routes the large Wikimedia-family ZIMs (Wikipedia,
+// Wikiversity, Wikibooks — including the flagship full Wikipedia) to
+// dumps.wikimedia.org, which returns HTTP 403 for a default `axios/x` (or empty)
+// User-Agent per Wikimedia's UA policy. Identify ourselves descriptively so
+// those downloads succeed.
+const DOWNLOAD_HEADERS: Record<string, string> = {
+  'User-Agent': 'ProjectNOMAD/1.0 (+https://projectnomad.us)',
+}
+
 /**
  * Perform a resumable download with progress tracking
  * @param param0 - Download parameters. Leave allowedMimeTypes empty to skip mime type checking.
@@ -45,6 +55,7 @@ export async function doResumableDownload({
   const headResponse = await axios.head(url, {
     signal,
     timeout,
+    headers: DOWNLOAD_HEADERS,
   })
 
   // Some upstream hosts (notably download.kiwix.org for .zim files) don't set a
@@ -94,7 +105,12 @@ export async function doResumableDownload({
   }
 
   const fetchStream = (hdrs: Record<string, string>) =>
-    axios.get(url, { responseType: 'stream', headers: hdrs, signal, timeout })
+    axios.get(url, {
+      responseType: 'stream',
+      headers: { ...DOWNLOAD_HEADERS, ...hdrs },
+      signal,
+      timeout,
+    })
 
   let response = await fetchStream(headers)