diff --git a/src/lib/components/functional/FormatDropdown.svelte b/src/lib/components/functional/FormatDropdown.svelte index 26c93cd..633b41e 100644 --- a/src/lib/components/functional/FormatDropdown.svelte +++ b/src/lib/components/functional/FormatDropdown.svelte @@ -265,7 +265,7 @@ const extract = async () => { // extract all files in zip, then add all extracted files to files store if (!file) return; - const { extractZip } = await import("$lib/util/zip"); + const { extractZip } = await import("$lib/util/file"); const extractedFiles = await extractZip(file.file); if (!Array.isArray(extractedFiles) || extractedFiles.length === 0) diff --git a/src/lib/converters/mediabunny.svelte.ts b/src/lib/converters/mediabunny.svelte.ts index a3c319f..a8d06b0 100644 --- a/src/lib/converters/mediabunny.svelte.ts +++ b/src/lib/converters/mediabunny.svelte.ts @@ -14,6 +14,7 @@ import { MpegTsOutputFormat, Output, QTFF, + StreamTarget, WEBM, WebMOutputFormat, } from "mediabunny"; @@ -33,24 +34,10 @@ import { browser } from "$app/environment"; // codec compatibility stuff, based on mediabunny's docs // https://mediabunny.dev/guide/supported-formats-and-codecs#compatibility-table +// prettier-ignore const mp4VideoCodecs = ["avc", "hevc", "vp8", "vp9", "av1"] as const; -const mp4AudioCodecs = [ - "aac", - "opus", - "mp3", - "vorbis", - "flac", - "ac3", - "eac3", - "pcm-s16", - "pcm-s16be", - "pcm-s24", - "pcm-s24be", - "pcm-s32", - "pcm-s32be", - "pcm-f32", - "pcm-f64", -] as const; +// prettier-ignore +const mp4AudioCodecs = [ "aac", "opus", "mp3", "vorbis", "flac", "ac3", "eac3", "pcm-s16", "pcm-s16be", "pcm-s24", "pcm-s24be", "pcm-s32", "pcm-s32be", "pcm-f32", "pcm-f64"] as const; const codecCompatibility = { video: { mp4: mp4VideoCodecs, @@ -194,6 +181,7 @@ export class MediabunnyConverter extends Converter { public reportsProgress: boolean = true; private activeConversions = new Map(); + private pendingOutputCleanups = new Map Promise>(); private formats: string[] = [ "mp4", @@ -477,15 +465,31 @@ export class MediabunnyConverter extends Converter { to: string, settings: ConversionSettings, ): Promise { + const toFormat = to.startsWith(".") ? to.slice(1) : to; + const originalName = file.file.name.split(".").slice(0, -1).join("."); + const outputFilename = `${originalName}.${toFormat}`; + const input = new Input({ // TODO: add settings & special handling for certain formats & codecs formats: [MP4, QTFF, MATROSKA, WEBM, MPEG_TS], source: new BlobSource(file.file), }); + const streamTargetContext = + await this.createStreamingTarget(outputFilename); + if (streamTargetContext) { + this.log(`using OPFS stream target for ${file.name}`); + this.pendingOutputCleanups.set( + file.id, + streamTargetContext.cleanup, + ); + } + + const target = streamTargetContext?.target ?? new BufferTarget(); + const output = new Output({ format: this.format(to), - target: new BufferTarget(), + target, }); const conversionSettings = @@ -546,22 +550,34 @@ export class MediabunnyConverter extends Converter { file.progress = progress * 100; }; - await conversion.execute(); - this.activeConversions.delete(file.id); + try { + await conversion.execute(); + } catch (err) { + const cleanup = this.pendingOutputCleanups.get(file.id); + if (cleanup) { + await cleanup(); + this.pendingOutputCleanups.delete(file.id); + } + throw err; + } finally { + this.activeConversions.delete(file.id); + } - if (!output.target.buffer) { + if (streamTargetContext) { + const streamedFile = await streamTargetContext.getFile(); + const result = new VertFile(streamedFile, toFormat); + result.setPostDownload(streamTargetContext.cleanup); + this.pendingOutputCleanups.delete(file.id); + return result; + } + + if (!(target instanceof BufferTarget) || !target.buffer) { throw new Error("Mediabunny conversion failed: no output buffer"); } - const toFormat = to.startsWith(".") ? to.slice(1) : to; - const originalName = file.file.name.split(".").slice(0, -1).join("."); - const f = new File( - [output.target.buffer], - `${originalName}.${toFormat}`, - { - type: "application/octet-stream", - }, - ); + const f = new File([target.buffer], `${originalName}.${toFormat}`, { + type: "application/octet-stream", + }); return new VertFile(f, toFormat); } @@ -599,5 +615,58 @@ export class MediabunnyConverter extends Converter { conversion.cancel(); this.activeConversions.delete(input.id); + + const cleanup = this.pendingOutputCleanups.get(input.id); + if (cleanup) { + await cleanup(); + this.pendingOutputCleanups.delete(input.id); + } + } + + private async createStreamingTarget(filename: string): Promise<{ + target: StreamTarget; + getFile: () => Promise; + cleanup: () => Promise; + } | null> { + try { + const storage = navigator.storage as StorageManager & { + getDirectory?: () => Promise; + }; + if (!storage.getDirectory) return null; + + const root = await storage.getDirectory(); + const tempDir = await root.getDirectoryHandle("vert-temp", { + create: true, + }); + const tempName = `${Date.now()}-${Math.random().toString(36).slice(2)}-${filename}`; + const fileHandle = await tempDir.getFileHandle(tempName, { + create: true, + }); + + const fileStream = await fileHandle.createWritable(); + const writable = new WritableStream({ + write: (chunk) => fileStream.write(chunk), + close: () => fileStream.close(), + abort: (reason) => fileStream.abort(reason), + }); + + const cleanup = async () => { + await tempDir.removeEntry(tempName).catch(() => {}); + }; + + return { + target: new StreamTarget(writable, { + chunked: true, + chunkSize: 32 * 1024 * 1024, + }), + getFile: () => fileHandle.getFile(), + cleanup, + }; + } catch (err) { + this.error( + `failed to initialize OPFS stream target, falling back to BufferTarget: ${err}`, + ); + return null; + } } } diff --git a/src/lib/store/index.svelte.ts b/src/lib/store/index.svelte.ts index 64077a3..54b5a89 100644 --- a/src/lib/store/index.svelte.ts +++ b/src/lib/store/index.svelte.ts @@ -12,6 +12,7 @@ import sanitizeHtml from "sanitize-html"; import { ToastManager } from "$lib/util/toast.svelte"; import { GB } from "$lib/util/consts"; import { readSettings } from "$lib/util/settings"; +import { formatFilename } from "$lib/util/file"; class Files { public files = $state([]); @@ -209,7 +210,7 @@ class Files { }), }); - const { extractZip } = await import("$lib/util/zip"); + const { extractZip } = await import("$lib/util/file"); const entries = await extractZip(file); const totalEntries = entries.length; @@ -439,7 +440,7 @@ class Files { dlFiles.push({ name: filename, lastModified: Date.now(), - input: await result.file.arrayBuffer(), + input: result.file.stream(), }); } @@ -450,17 +451,9 @@ class Files { const settings = readSettings<{ filenameFormat?: string }>(); const filenameFormat = settings.filenameFormat || "VERT_%name%"; - const format = (name: string) => { - const date = new Date().toISOString(); - return name - .replace(/%date%/g, date) - .replace(/%name%/g, "Multi") - .replace(/%extension%/g, ""); - }; - const a = document.createElement("a"); a.href = url; - a.download = `${format(filenameFormat)}.zip`; + a.download = `${formatFilename(filenameFormat, "Multi")}.zip`; a.click(); URL.revokeObjectURL(url); a.remove(); diff --git a/src/lib/types/file.svelte.ts b/src/lib/types/file.svelte.ts index f006ad2..9154cf4 100644 --- a/src/lib/types/file.svelte.ts +++ b/src/lib/types/file.svelte.ts @@ -12,8 +12,9 @@ import type { } from "./conversion-settings"; import { log } from "$lib/util/logger"; import { readSettings } from "$lib/util/settings"; +import { formatFilename } from "$lib/util/file"; -const MAX_BLOB_SIZE_LIMIT = 2 * 1024 * 1024 * 1024; // 2GB +const LARGE_FILE = 2 * 1024 * 1024 * 1024; // 2GB export class VertFile { public id: string = Math.random().toString(36).slice(2, 8); @@ -44,9 +45,26 @@ export class VertFile { private attemptedConverters = new Set(); private retryingFallback = false; private vertdWarningToastId: number | null = null; + private postDownload: (() => Promise) | null = null; public isZip = $state(() => this.from === ".zip"); + public setPostDownload(cleanup: (() => Promise) | null) { + this.postDownload = cleanup; + } + + private async runPostDownload() { + if (!this.postDownload) return; + + try { + await this.postDownload(); + } catch (err) { + log(["file", "cleanup"], `post-download function failed: ${err}`); + } finally { + this.postDownload = null; + } + } + public getAvailableSettings( input: VertFile, converter: string | undefined = this.conversionSettings.converter, @@ -109,13 +127,19 @@ export class VertFile { } public supportsStreaming(): boolean { - // only vertd (video/gif -> video/gif) supports streaming - // rest of converters need entire file in memory, limited by ArrayBuffer limits + // vertd supports server-side streaming; mediabunny can stream to OPFS if available + const opfsSupported = + typeof navigator !== "undefined" && + "storage" in navigator && + typeof navigator.storage.getDirectory === "function"; + const availableConverters = this.isZip() ? this.converters : this.findConverters(); return availableConverters.some( - (converter) => converter.name === "vertd", + (converter) => + converter.name === "vertd" || + (converter.name === "mediabunny" && opfsSupported), ); } @@ -144,6 +168,8 @@ export class VertFile { // eslint-disable-next-line @typescript-eslint/no-explicit-any public async convert(...args: any[]) { + await this.runPostDownload(); + if (!this.retryingFallback) this.attemptedConverters.clear(); if (!this.converters.length) throw new Error("No converters found"); @@ -323,7 +349,7 @@ export class VertFile { } private async convertZip(converter: Converter): Promise { - const { extractZip, createZip } = await import("$lib/util/zip"); + const { extractZip, createZip } = await import("$lib/util/file"); const { default: PQueue } = await import("p-queue"); const entries = await extractZip(this.file); @@ -483,61 +509,55 @@ export class VertFile { const settings = readSettings<{ filenameFormat?: string }>(); const filenameFormat = settings.filenameFormat || "VERT_%name%"; - const format = (name: string) => { - const now = new Date(); - const iso = now.toISOString(); - const date = iso.split("T")[0]; - const time = iso.split("T")[1].split(".")[0].replace(/:/g, "-"); - const unix = now.getTime().toString(); - const baseName = this.file.name.replace(/\.[^/.]+$/, ""); - const originalExtension = this.file.name.split(".").pop()!; - return name - .replace(/%datetime%/g, iso) - .replace(/%date%/g, date) - .replace(/%time%/g, time) - .replace(/%unix%/g, unix) - .replace(/%name%/g, baseName) - .replace(/%extension%/g, originalExtension); + const filename = `${formatFilename(filenameFormat, this.file)}${to}`; + const resultFile = this.result.file; + + const filePicker = window as Window & { + showSaveFilePicker?: (options?: { + suggestedName?: string; + types?: Array<{ + description?: string; + accept: Record; + }>; + }) => Promise; }; - const filename = `${format(filenameFormat)}${to}`; + const diskStreamSupported = + typeof filePicker.showSaveFilePicker === "function"; + const shouldDiskStream = + diskStreamSupported && resultFile.size >= LARGE_FILE; - // larger files (>2gb) requires cache API - // blob constructor can't use arraybuffer above 2gb - const useCacheApi = this.result.file.size > MAX_BLOB_SIZE_LIMIT; - let blob: string; + if (shouldDiskStream) { + // use the File System Access API to directly stream to disk, so we can actually save larger files + try { + const ext = to.slice(1); + const handle = await filePicker.showSaveFilePicker!({ + suggestedName: filename, + types: [ + { + description: "The VERT converted file", + accept: { "application/octet-stream": [`.${ext}`] }, + }, + ], + }); - if (useCacheApi) { - const cache = await caches.open("vert-downloads"); - const cacheKey = `vert-download-${Date.now()}-${filename}`; - - const response = new Response(this.result.file.stream(), { - headers: { - "Content-Type": "application/octet-stream", - "Content-Length": this.result.file.size.toString(), - }, - }); - - await cache.put(cacheKey, response); - - const cachedResponse = await cache.match(cacheKey); - if (!cachedResponse) - throw new Error("Failed to cache file for download"); - - const cachedBlob = await cachedResponse.blob(); - blob = URL.createObjectURL(cachedBlob); - - setTimeout(() => { - cache.delete(cacheKey); - }, 30000); - } else { - blob = URL.createObjectURL( - new Blob([await this.result.file.arrayBuffer()], { - type: "application/octet-stream", - }), - ); + const writable = await handle.createWritable(); + await resultFile.stream().pipeTo(writable); + this.blobUrl = undefined; + return; + } catch (err) { + const casted = err as DOMException; + if (casted?.name === "AbortError") return; + log( + ["file", "download"], + `disk-streaming download failed, falling back to blob URL: ${err}`, + ); + } } + // fallback to blob URL download for smaller files or if the File System Access API isn't supported + const blob = URL.createObjectURL(resultFile); + // download const a = document.createElement("a"); a.href = blob; diff --git a/src/lib/util/zip.ts b/src/lib/util/file.ts similarity index 61% rename from src/lib/util/zip.ts rename to src/lib/util/file.ts index 8ff9c46..bb9dd6f 100644 --- a/src/lib/util/zip.ts +++ b/src/lib/util/file.ts @@ -28,7 +28,10 @@ export async function extractZip(file: File): Promise { data: new Uint8Array(data), })); - log(["zip"], `extracted ${entries.length} entries from ${file.name}`); + log( + ["zip"], + `extracted ${entries.length} entries from ${file.name}`, + ); resolve(entries); }); }); @@ -47,3 +50,27 @@ export function ignoreEntry(filename: string): boolean { filename.endsWith("/") ); } + +export function formatFilename(format: string, file: File | string) { + const now = new Date(); + const iso = now.toISOString(); + const date = iso.split("T")[0]; + const time = iso.split("T")[1].split(".")[0].replace(/:/g, "-"); + const unix = now.getTime().toString(); + const baseName = + typeof file === "string" + ? file.replace(/\.[^/.]+$/, "") + : file.name.replace(/\.[^/.]+$/, ""); + const originalExtension = + typeof file === "string" + ? file.split(".").pop()! + : file.name.split(".").pop()!; + + return format + .replace(/%datetime%/g, iso) + .replace(/%date%/g, date) + .replace(/%time%/g, time) + .replace(/%unix%/g, unix) + .replace(/%name%/g, baseName) + .replace(/%extension%/g, originalExtension); +}