fix: large file fixes

mediabunny reads streamed file from OPFS to allow for larger files, larger files write directly to disk instead of cache api

src/lib/components/functional/FormatDropdown.svelte

@@ -265,7 +265,7 @@
 	const extract = async () => {
 		// extract all files in zip, then add all extracted files to files store
 		if (!file) return;
-		const { extractZip } = await import("$lib/util/zip");
+		const { extractZip } = await import("$lib/util/file");
 		const extractedFiles = await extractZip(file.file);
 
 		if (!Array.isArray(extractedFiles) || extractedFiles.length === 0)

src/lib/converters/mediabunny.svelte.ts

@@ -14,6 +14,7 @@ import {
 	MpegTsOutputFormat,
 	Output,
 	QTFF,
+	StreamTarget,
 	WEBM,
 	WebMOutputFormat,
 } from "mediabunny";
@@ -33,24 +34,10 @@ import { browser } from "$app/environment";
 
 // codec compatibility stuff, based on mediabunny's docs
 // https://mediabunny.dev/guide/supported-formats-and-codecs#compatibility-table
+// prettier-ignore
 const mp4VideoCodecs = ["avc", "hevc", "vp8", "vp9", "av1"] as const;
-const mp4AudioCodecs = [
+// prettier-ignore
-	"aac",
+const mp4AudioCodecs = [ "aac", "opus", "mp3", "vorbis", "flac", "ac3", "eac3", "pcm-s16", "pcm-s16be", "pcm-s24", "pcm-s24be", "pcm-s32", "pcm-s32be", "pcm-f32", "pcm-f64"] as const;
-	"opus",
-	"mp3",
-	"vorbis",
-	"flac",
-	"ac3",
-	"eac3",
-	"pcm-s16",
-	"pcm-s16be",
-	"pcm-s24",
-	"pcm-s24be",
-	"pcm-s32",
-	"pcm-s32be",
-	"pcm-f32",
-	"pcm-f64",
-] as const;
 const codecCompatibility = {
 	video: {
 		mp4: mp4VideoCodecs,
@@ -194,6 +181,7 @@ export class MediabunnyConverter extends Converter {
 	public reportsProgress: boolean = true;
 
 	private activeConversions = new Map<string, Conversion>();
+	private pendingOutputCleanups = new Map<string, () => Promise<void>>();
 
 	private formats: string[] = [
 		"mp4",
@@ -477,15 +465,31 @@ export class MediabunnyConverter extends Converter {
 		to: string,
 		settings: ConversionSettings,
 	): Promise<VertFile> {
+		const toFormat = to.startsWith(".") ? to.slice(1) : to;
+		const originalName = file.file.name.split(".").slice(0, -1).join(".");
+		const outputFilename = `${originalName}.${toFormat}`;
+
 		const input = new Input({
 			// TODO: add settings & special handling for certain formats & codecs
 			formats: [MP4, QTFF, MATROSKA, WEBM, MPEG_TS],
 			source: new BlobSource(file.file),
 		});
 
+		const streamTargetContext =
+			await this.createStreamingTarget(outputFilename);
+		if (streamTargetContext) {
+			this.log(`using OPFS stream target for ${file.name}`);
+			this.pendingOutputCleanups.set(
+				file.id,
+				streamTargetContext.cleanup,
+			);
+		}
+
+		const target = streamTargetContext?.target ?? new BufferTarget();
+
 		const output = new Output({
 			format: this.format(to),
-			target: new BufferTarget(),
+			target,
 		});
 
 		const conversionSettings =
@@ -546,22 +550,34 @@ export class MediabunnyConverter extends Converter {
 			file.progress = progress * 100;
 		};
 
-		await conversion.execute();
+		try {
-		this.activeConversions.delete(file.id);
+			await conversion.execute();
+		} catch (err) {
+			const cleanup = this.pendingOutputCleanups.get(file.id);
+			if (cleanup) {
+				await cleanup();
+				this.pendingOutputCleanups.delete(file.id);
+			}
+			throw err;
+		} finally {
+			this.activeConversions.delete(file.id);
+		}
 
-		if (!output.target.buffer) {
+		if (streamTargetContext) {
+			const streamedFile = await streamTargetContext.getFile();
+			const result = new VertFile(streamedFile, toFormat);
+			result.setPostDownload(streamTargetContext.cleanup);
+			this.pendingOutputCleanups.delete(file.id);
+			return result;
+		}
+
+		if (!(target instanceof BufferTarget) || !target.buffer) {
 			throw new Error("Mediabunny conversion failed: no output buffer");
 		}
 
-		const toFormat = to.startsWith(".") ? to.slice(1) : to;
+		const f = new File([target.buffer], `${originalName}.${toFormat}`, {
-		const originalName = file.file.name.split(".").slice(0, -1).join(".");
+			type: "application/octet-stream",
-		const f = new File(
+		});
-			[output.target.buffer],
-			`${originalName}.${toFormat}`,
-			{
-				type: "application/octet-stream",
-			},
-		);
 
 		return new VertFile(f, toFormat);
 	}
@@ -599,5 +615,58 @@ export class MediabunnyConverter extends Converter {
 
 		conversion.cancel();
 		this.activeConversions.delete(input.id);
+
+		const cleanup = this.pendingOutputCleanups.get(input.id);
+		if (cleanup) {
+			await cleanup();
+			this.pendingOutputCleanups.delete(input.id);
+		}
+	}
+
+	private async createStreamingTarget(filename: string): Promise<{
+		target: StreamTarget;
+		getFile: () => Promise<File>;
+		cleanup: () => Promise<void>;
+	} | null> {
+		try {
+			const storage = navigator.storage as StorageManager & {
+				getDirectory?: () => Promise<FileSystemDirectoryHandle>;
+			};
+			if (!storage.getDirectory) return null;
+
+			const root = await storage.getDirectory();
+			const tempDir = await root.getDirectoryHandle("vert-temp", {
+				create: true,
+			});
+			const tempName = `${Date.now()}-${Math.random().toString(36).slice(2)}-${filename}`;
+			const fileHandle = await tempDir.getFileHandle(tempName, {
+				create: true,
+			});
+
+			const fileStream = await fileHandle.createWritable();
+			const writable = new WritableStream({
+				write: (chunk) => fileStream.write(chunk),
+				close: () => fileStream.close(),
+				abort: (reason) => fileStream.abort(reason),
+			});
+
+			const cleanup = async () => {
+				await tempDir.removeEntry(tempName).catch(() => {});
+			};
+
+			return {
+				target: new StreamTarget(writable, {
+					chunked: true,
+					chunkSize: 32 * 1024 * 1024,
+				}),
+				getFile: () => fileHandle.getFile(),
+				cleanup,
+			};
+		} catch (err) {
+			this.error(
+				`failed to initialize OPFS stream target, falling back to BufferTarget: ${err}`,
+			);
+			return null;
+		}
 	}
 }

src/lib/store/index.svelte.ts

@@ -12,6 +12,7 @@ import sanitizeHtml from "sanitize-html";
 import { ToastManager } from "$lib/util/toast.svelte";
 import { GB } from "$lib/util/consts";
 import { readSettings } from "$lib/util/settings";
+import { formatFilename } from "$lib/util/file";
 
 class Files {
 	public files = $state<VertFile[]>([]);
@@ -209,7 +210,7 @@ class Files {
 				}),
 			});
 
-			const { extractZip } = await import("$lib/util/zip");
+			const { extractZip } = await import("$lib/util/file");
 			const entries = await extractZip(file);
 
 			const totalEntries = entries.length;
@@ -439,7 +440,7 @@ class Files {
 			dlFiles.push({
 				name: filename,
 				lastModified: Date.now(),
-				input: await result.file.arrayBuffer(),
+				input: result.file.stream(),
 			});
 		}
 
@@ -450,17 +451,9 @@ class Files {
 		const settings = readSettings<{ filenameFormat?: string }>();
 		const filenameFormat = settings.filenameFormat || "VERT_%name%";
 
-		const format = (name: string) => {
-			const date = new Date().toISOString();
-			return name
-				.replace(/%date%/g, date)
-				.replace(/%name%/g, "Multi")
-				.replace(/%extension%/g, "");
-		};
-
 		const a = document.createElement("a");
 		a.href = url;
-		a.download = `${format(filenameFormat)}.zip`;
+		a.download = `${formatFilename(filenameFormat, "Multi")}.zip`;
 		a.click();
 		URL.revokeObjectURL(url);
 		a.remove();

src/lib/types/file.svelte.ts

@@ -12,8 +12,9 @@ import type {
 } from "./conversion-settings";
 import { log } from "$lib/util/logger";
 import { readSettings } from "$lib/util/settings";
+import { formatFilename } from "$lib/util/file";
 
-const MAX_BLOB_SIZE_LIMIT = 2 * 1024 * 1024 * 1024; // 2GB
+const LARGE_FILE = 2 * 1024 * 1024 * 1024; // 2GB
 
 export class VertFile {
 	public id: string = Math.random().toString(36).slice(2, 8);
@@ -44,9 +45,26 @@ export class VertFile {
 	private attemptedConverters = new Set<string>();
 	private retryingFallback = false;
 	private vertdWarningToastId: number | null = null;
+	private postDownload: (() => Promise<void>) | null = null;
 
 	public isZip = $state(() => this.from === ".zip");
 
+	public setPostDownload(cleanup: (() => Promise<void>) | null) {
+		this.postDownload = cleanup;
+	}
+
+	private async runPostDownload() {
+		if (!this.postDownload) return;
+
+		try {
+			await this.postDownload();
+		} catch (err) {
+			log(["file", "cleanup"], `post-download function failed: ${err}`);
+		} finally {
+			this.postDownload = null;
+		}
+	}
+
 	public getAvailableSettings(
 		input: VertFile,
 		converter: string | undefined = this.conversionSettings.converter,
@@ -109,13 +127,19 @@ export class VertFile {
 	}
 
 	public supportsStreaming(): boolean {
-		// only vertd (video/gif -> video/gif) supports streaming
+		// vertd supports server-side streaming; mediabunny can stream to OPFS if available
-		// rest of converters need entire file in memory, limited by ArrayBuffer limits
+		const opfsSupported =
+			typeof navigator !== "undefined" &&
+			"storage" in navigator &&
+			typeof navigator.storage.getDirectory === "function";
+
 		const availableConverters = this.isZip()
 			? this.converters
 			: this.findConverters();
 		return availableConverters.some(
-			(converter) => converter.name === "vertd",
+			(converter) =>
+				converter.name === "vertd" ||
+				(converter.name === "mediabunny" && opfsSupported),
 		);
 	}
 
@@ -144,6 +168,8 @@ export class VertFile {
 
 	// eslint-disable-next-line @typescript-eslint/no-explicit-any
 	public async convert(...args: any[]) {
+		await this.runPostDownload();
+
 		if (!this.retryingFallback) this.attemptedConverters.clear();
 
 		if (!this.converters.length) throw new Error("No converters found");
@@ -323,7 +349,7 @@ export class VertFile {
 	}
 
 	private async convertZip(converter: Converter): Promise<VertFile> {
-		const { extractZip, createZip } = await import("$lib/util/zip");
+		const { extractZip, createZip } = await import("$lib/util/file");
 		const { default: PQueue } = await import("p-queue");
 
 		const entries = await extractZip(this.file);
@@ -483,61 +509,55 @@ export class VertFile {
 		const settings = readSettings<{ filenameFormat?: string }>();
 		const filenameFormat = settings.filenameFormat || "VERT_%name%";
 
-		const format = (name: string) => {
+		const filename = `${formatFilename(filenameFormat, this.file)}${to}`;
-			const now = new Date();
+		const resultFile = this.result.file;
-			const iso = now.toISOString();
+
-			const date = iso.split("T")[0];
+		const filePicker = window as Window & {
-			const time = iso.split("T")[1].split(".")[0].replace(/:/g, "-");
+			showSaveFilePicker?: (options?: {
-			const unix = now.getTime().toString();
+				suggestedName?: string;
-			const baseName = this.file.name.replace(/\.[^/.]+$/, "");
+				types?: Array<{
-			const originalExtension = this.file.name.split(".").pop()!;
+					description?: string;
-			return name
+					accept: Record<string, string[]>;
-				.replace(/%datetime%/g, iso)
+				}>;
-				.replace(/%date%/g, date)
+			}) => Promise<FileSystemFileHandle>;
-				.replace(/%time%/g, time)
-				.replace(/%unix%/g, unix)
-				.replace(/%name%/g, baseName)
-				.replace(/%extension%/g, originalExtension);
 		};
 
-		const filename = `${format(filenameFormat)}${to}`;
+		const diskStreamSupported =
+			typeof filePicker.showSaveFilePicker === "function";
+		const shouldDiskStream =
+			diskStreamSupported && resultFile.size >= LARGE_FILE;
 
-		// larger files (>2gb) requires cache API
+		if (shouldDiskStream) {
-		// blob constructor can't use arraybuffer above 2gb
+			// use the File System Access API to directly stream to disk, so we can actually save larger files
-		const useCacheApi = this.result.file.size > MAX_BLOB_SIZE_LIMIT;
+			try {
-		let blob: string;
+				const ext = to.slice(1);
+				const handle = await filePicker.showSaveFilePicker!({
+					suggestedName: filename,
+					types: [
+						{
+							description: "The VERT converted file",
+							accept: { "application/octet-stream": [`.${ext}`] },
+						},
+					],
+				});
 
-		if (useCacheApi) {
+				const writable = await handle.createWritable();
-			const cache = await caches.open("vert-downloads");
+				await resultFile.stream().pipeTo(writable);
-			const cacheKey = `vert-download-${Date.now()}-${filename}`;
+				this.blobUrl = undefined;
-
+				return;
-			const response = new Response(this.result.file.stream(), {
+			} catch (err) {
-				headers: {
+				const casted = err as DOMException;
-					"Content-Type": "application/octet-stream",
+				if (casted?.name === "AbortError") return;
-					"Content-Length": this.result.file.size.toString(),
+				log(
-				},
+					["file", "download"],
-			});
+					`disk-streaming download failed, falling back to blob URL: ${err}`,
-
+				);
-			await cache.put(cacheKey, response);
+			}
-
-			const cachedResponse = await cache.match(cacheKey);
-			if (!cachedResponse)
-				throw new Error("Failed to cache file for download");
-
-			const cachedBlob = await cachedResponse.blob();
-			blob = URL.createObjectURL(cachedBlob);
-
-			setTimeout(() => {
-				cache.delete(cacheKey);
-			}, 30000);
-		} else {
-			blob = URL.createObjectURL(
-				new Blob([await this.result.file.arrayBuffer()], {
-					type: "application/octet-stream",
-				}),
-			);
 		}
 
+		// fallback to blob URL download for smaller files or if the File System Access API isn't supported
+		const blob = URL.createObjectURL(resultFile);
+
 		// download
 		const a = document.createElement("a");
 		a.href = blob;

src/lib/util/file.ts

@@ -28,7 +28,10 @@ export async function extractZip(file: File): Promise<ZipEntry[]> {
 					data: new Uint8Array(data),
 				}));
 
-			log(["zip"], `extracted ${entries.length} entries from ${file.name}`);
+			log(
+				["zip"],
+				`extracted ${entries.length} entries from ${file.name}`,
+			);
 			resolve(entries);
 		});
 	});
@@ -47,3 +50,27 @@ export function ignoreEntry(filename: string): boolean {
 		filename.endsWith("/")
 	);
 }
+
+export function formatFilename(format: string, file: File | string) {
+	const now = new Date();
+	const iso = now.toISOString();
+	const date = iso.split("T")[0];
+	const time = iso.split("T")[1].split(".")[0].replace(/:/g, "-");
+	const unix = now.getTime().toString();
+	const baseName =
+		typeof file === "string"
+			? file.replace(/\.[^/.]+$/, "")
+			: file.name.replace(/\.[^/.]+$/, "");
+	const originalExtension =
+		typeof file === "string"
+			? file.split(".").pop()!
+			: file.name.split(".").pop()!;
+
+	return format
+		.replace(/%datetime%/g, iso)
+		.replace(/%date%/g, date)
+		.replace(/%time%/g, time)
+		.replace(/%unix%/g, unix)
+		.replace(/%name%/g, baseName)
+		.replace(/%extension%/g, originalExtension);
+}