mirror of https://github.com/VERT-sh/VERT.git
fix: large file fixes
mediabunny reads streamed file from OPFS to allow for larger files, larger files write directly to disk instead of cache api
This commit is contained in:
Maya
parent
d3aeb9b696
commit
75e9745eab
|
|
@ -265,7 +265,7 @@
|
|||
const extract = async () => {
|
||||
// extract all files in zip, then add all extracted files to files store
|
||||
if (!file) return;
|
||||
const { extractZip } = await import("$lib/util/zip");
|
||||
const { extractZip } = await import("$lib/util/file");
|
||||
const extractedFiles = await extractZip(file.file);
|
||||
|
||||
if (!Array.isArray(extractedFiles) || extractedFiles.length === 0)
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import {
|
|||
MpegTsOutputFormat,
|
||||
Output,
|
||||
QTFF,
|
||||
StreamTarget,
|
||||
WEBM,
|
||||
WebMOutputFormat,
|
||||
} from "mediabunny";
|
||||
|
|
@ -33,24 +34,10 @@ import { browser } from "$app/environment";
|
|||
|
||||
// codec compatibility stuff, based on mediabunny's docs
|
||||
// https://mediabunny.dev/guide/supported-formats-and-codecs#compatibility-table
|
||||
// prettier-ignore
|
||||
const mp4VideoCodecs = ["avc", "hevc", "vp8", "vp9", "av1"] as const;
|
||||
const mp4AudioCodecs = [
|
||||
"aac",
|
||||
"opus",
|
||||
"mp3",
|
||||
"vorbis",
|
||||
"flac",
|
||||
"ac3",
|
||||
"eac3",
|
||||
"pcm-s16",
|
||||
"pcm-s16be",
|
||||
"pcm-s24",
|
||||
"pcm-s24be",
|
||||
"pcm-s32",
|
||||
"pcm-s32be",
|
||||
"pcm-f32",
|
||||
"pcm-f64",
|
||||
] as const;
|
||||
// prettier-ignore
|
||||
const mp4AudioCodecs = [ "aac", "opus", "mp3", "vorbis", "flac", "ac3", "eac3", "pcm-s16", "pcm-s16be", "pcm-s24", "pcm-s24be", "pcm-s32", "pcm-s32be", "pcm-f32", "pcm-f64"] as const;
|
||||
const codecCompatibility = {
|
||||
video: {
|
||||
mp4: mp4VideoCodecs,
|
||||
|
|
@ -194,6 +181,7 @@ export class MediabunnyConverter extends Converter {
|
|||
public reportsProgress: boolean = true;
|
||||
|
||||
private activeConversions = new Map<string, Conversion>();
|
||||
private pendingOutputCleanups = new Map<string, () => Promise<void>>();
|
||||
|
||||
private formats: string[] = [
|
||||
"mp4",
|
||||
|
|
@ -477,15 +465,31 @@ export class MediabunnyConverter extends Converter {
|
|||
to: string,
|
||||
settings: ConversionSettings,
|
||||
): Promise<VertFile> {
|
||||
const toFormat = to.startsWith(".") ? to.slice(1) : to;
|
||||
const originalName = file.file.name.split(".").slice(0, -1).join(".");
|
||||
const outputFilename = `${originalName}.${toFormat}`;
|
||||
|
||||
const input = new Input({
|
||||
// TODO: add settings & special handling for certain formats & codecs
|
||||
formats: [MP4, QTFF, MATROSKA, WEBM, MPEG_TS],
|
||||
source: new BlobSource(file.file),
|
||||
});
|
||||
|
||||
const streamTargetContext =
|
||||
await this.createStreamingTarget(outputFilename);
|
||||
if (streamTargetContext) {
|
||||
this.log(`using OPFS stream target for ${file.name}`);
|
||||
this.pendingOutputCleanups.set(
|
||||
file.id,
|
||||
streamTargetContext.cleanup,
|
||||
);
|
||||
}
|
||||
|
||||
const target = streamTargetContext?.target ?? new BufferTarget();
|
||||
|
||||
const output = new Output({
|
||||
format: this.format(to),
|
||||
target: new BufferTarget(),
|
||||
target,
|
||||
});
|
||||
|
||||
const conversionSettings =
|
||||
|
|
@ -546,22 +550,34 @@ export class MediabunnyConverter extends Converter {
|
|||
file.progress = progress * 100;
|
||||
};
|
||||
|
||||
await conversion.execute();
|
||||
this.activeConversions.delete(file.id);
|
||||
try {
|
||||
await conversion.execute();
|
||||
} catch (err) {
|
||||
const cleanup = this.pendingOutputCleanups.get(file.id);
|
||||
if (cleanup) {
|
||||
await cleanup();
|
||||
this.pendingOutputCleanups.delete(file.id);
|
||||
}
|
||||
throw err;
|
||||
} finally {
|
||||
this.activeConversions.delete(file.id);
|
||||
}
|
||||
|
||||
if (!output.target.buffer) {
|
||||
if (streamTargetContext) {
|
||||
const streamedFile = await streamTargetContext.getFile();
|
||||
const result = new VertFile(streamedFile, toFormat);
|
||||
result.setPostDownload(streamTargetContext.cleanup);
|
||||
this.pendingOutputCleanups.delete(file.id);
|
||||
return result;
|
||||
}
|
||||
|
||||
if (!(target instanceof BufferTarget) || !target.buffer) {
|
||||
throw new Error("Mediabunny conversion failed: no output buffer");
|
||||
}
|
||||
|
||||
const toFormat = to.startsWith(".") ? to.slice(1) : to;
|
||||
const originalName = file.file.name.split(".").slice(0, -1).join(".");
|
||||
const f = new File(
|
||||
[output.target.buffer],
|
||||
`${originalName}.${toFormat}`,
|
||||
{
|
||||
type: "application/octet-stream",
|
||||
},
|
||||
);
|
||||
const f = new File([target.buffer], `${originalName}.${toFormat}`, {
|
||||
type: "application/octet-stream",
|
||||
});
|
||||
|
||||
return new VertFile(f, toFormat);
|
||||
}
|
||||
|
|
@ -599,5 +615,58 @@ export class MediabunnyConverter extends Converter {
|
|||
|
||||
conversion.cancel();
|
||||
this.activeConversions.delete(input.id);
|
||||
|
||||
const cleanup = this.pendingOutputCleanups.get(input.id);
|
||||
if (cleanup) {
|
||||
await cleanup();
|
||||
this.pendingOutputCleanups.delete(input.id);
|
||||
}
|
||||
}
|
||||
|
||||
private async createStreamingTarget(filename: string): Promise<{
|
||||
target: StreamTarget;
|
||||
getFile: () => Promise<File>;
|
||||
cleanup: () => Promise<void>;
|
||||
} | null> {
|
||||
try {
|
||||
const storage = navigator.storage as StorageManager & {
|
||||
getDirectory?: () => Promise<FileSystemDirectoryHandle>;
|
||||
};
|
||||
if (!storage.getDirectory) return null;
|
||||
|
||||
const root = await storage.getDirectory();
|
||||
const tempDir = await root.getDirectoryHandle("vert-temp", {
|
||||
create: true,
|
||||
});
|
||||
const tempName = `${Date.now()}-${Math.random().toString(36).slice(2)}-${filename}`;
|
||||
const fileHandle = await tempDir.getFileHandle(tempName, {
|
||||
create: true,
|
||||
});
|
||||
|
||||
const fileStream = await fileHandle.createWritable();
|
||||
const writable = new WritableStream({
|
||||
write: (chunk) => fileStream.write(chunk),
|
||||
close: () => fileStream.close(),
|
||||
abort: (reason) => fileStream.abort(reason),
|
||||
});
|
||||
|
||||
const cleanup = async () => {
|
||||
await tempDir.removeEntry(tempName).catch(() => {});
|
||||
};
|
||||
|
||||
return {
|
||||
target: new StreamTarget(writable, {
|
||||
chunked: true,
|
||||
chunkSize: 32 * 1024 * 1024,
|
||||
}),
|
||||
getFile: () => fileHandle.getFile(),
|
||||
cleanup,
|
||||
};
|
||||
} catch (err) {
|
||||
this.error(
|
||||
`failed to initialize OPFS stream target, falling back to BufferTarget: ${err}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import sanitizeHtml from "sanitize-html";
|
|||
import { ToastManager } from "$lib/util/toast.svelte";
|
||||
import { GB } from "$lib/util/consts";
|
||||
import { readSettings } from "$lib/util/settings";
|
||||
import { formatFilename } from "$lib/util/file";
|
||||
|
||||
class Files {
|
||||
public files = $state<VertFile[]>([]);
|
||||
|
|
@ -209,7 +210,7 @@ class Files {
|
|||
}),
|
||||
});
|
||||
|
||||
const { extractZip } = await import("$lib/util/zip");
|
||||
const { extractZip } = await import("$lib/util/file");
|
||||
const entries = await extractZip(file);
|
||||
|
||||
const totalEntries = entries.length;
|
||||
|
|
@ -439,7 +440,7 @@ class Files {
|
|||
dlFiles.push({
|
||||
name: filename,
|
||||
lastModified: Date.now(),
|
||||
input: await result.file.arrayBuffer(),
|
||||
input: result.file.stream(),
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -450,17 +451,9 @@ class Files {
|
|||
const settings = readSettings<{ filenameFormat?: string }>();
|
||||
const filenameFormat = settings.filenameFormat || "VERT_%name%";
|
||||
|
||||
const format = (name: string) => {
|
||||
const date = new Date().toISOString();
|
||||
return name
|
||||
.replace(/%date%/g, date)
|
||||
.replace(/%name%/g, "Multi")
|
||||
.replace(/%extension%/g, "");
|
||||
};
|
||||
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
a.download = `${format(filenameFormat)}.zip`;
|
||||
a.download = `${formatFilename(filenameFormat, "Multi")}.zip`;
|
||||
a.click();
|
||||
URL.revokeObjectURL(url);
|
||||
a.remove();
|
||||
|
|
|
|||
|
|
@ -12,8 +12,9 @@ import type {
|
|||
} from "./conversion-settings";
|
||||
import { log } from "$lib/util/logger";
|
||||
import { readSettings } from "$lib/util/settings";
|
||||
import { formatFilename } from "$lib/util/file";
|
||||
|
||||
const MAX_BLOB_SIZE_LIMIT = 2 * 1024 * 1024 * 1024; // 2GB
|
||||
const LARGE_FILE = 2 * 1024 * 1024 * 1024; // 2GB
|
||||
|
||||
export class VertFile {
|
||||
public id: string = Math.random().toString(36).slice(2, 8);
|
||||
|
|
@ -44,9 +45,26 @@ export class VertFile {
|
|||
private attemptedConverters = new Set<string>();
|
||||
private retryingFallback = false;
|
||||
private vertdWarningToastId: number | null = null;
|
||||
private postDownload: (() => Promise<void>) | null = null;
|
||||
|
||||
public isZip = $state(() => this.from === ".zip");
|
||||
|
||||
public setPostDownload(cleanup: (() => Promise<void>) | null) {
|
||||
this.postDownload = cleanup;
|
||||
}
|
||||
|
||||
private async runPostDownload() {
|
||||
if (!this.postDownload) return;
|
||||
|
||||
try {
|
||||
await this.postDownload();
|
||||
} catch (err) {
|
||||
log(["file", "cleanup"], `post-download function failed: ${err}`);
|
||||
} finally {
|
||||
this.postDownload = null;
|
||||
}
|
||||
}
|
||||
|
||||
public getAvailableSettings(
|
||||
input: VertFile,
|
||||
converter: string | undefined = this.conversionSettings.converter,
|
||||
|
|
@ -109,13 +127,19 @@ export class VertFile {
|
|||
}
|
||||
|
||||
public supportsStreaming(): boolean {
|
||||
// only vertd (video/gif -> video/gif) supports streaming
|
||||
// rest of converters need entire file in memory, limited by ArrayBuffer limits
|
||||
// vertd supports server-side streaming; mediabunny can stream to OPFS if available
|
||||
const opfsSupported =
|
||||
typeof navigator !== "undefined" &&
|
||||
"storage" in navigator &&
|
||||
typeof navigator.storage.getDirectory === "function";
|
||||
|
||||
const availableConverters = this.isZip()
|
||||
? this.converters
|
||||
: this.findConverters();
|
||||
return availableConverters.some(
|
||||
(converter) => converter.name === "vertd",
|
||||
(converter) =>
|
||||
converter.name === "vertd" ||
|
||||
(converter.name === "mediabunny" && opfsSupported),
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -144,6 +168,8 @@ export class VertFile {
|
|||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
public async convert(...args: any[]) {
|
||||
await this.runPostDownload();
|
||||
|
||||
if (!this.retryingFallback) this.attemptedConverters.clear();
|
||||
|
||||
if (!this.converters.length) throw new Error("No converters found");
|
||||
|
|
@ -323,7 +349,7 @@ export class VertFile {
|
|||
}
|
||||
|
||||
private async convertZip(converter: Converter): Promise<VertFile> {
|
||||
const { extractZip, createZip } = await import("$lib/util/zip");
|
||||
const { extractZip, createZip } = await import("$lib/util/file");
|
||||
const { default: PQueue } = await import("p-queue");
|
||||
|
||||
const entries = await extractZip(this.file);
|
||||
|
|
@ -483,61 +509,55 @@ export class VertFile {
|
|||
const settings = readSettings<{ filenameFormat?: string }>();
|
||||
const filenameFormat = settings.filenameFormat || "VERT_%name%";
|
||||
|
||||
const format = (name: string) => {
|
||||
const now = new Date();
|
||||
const iso = now.toISOString();
|
||||
const date = iso.split("T")[0];
|
||||
const time = iso.split("T")[1].split(".")[0].replace(/:/g, "-");
|
||||
const unix = now.getTime().toString();
|
||||
const baseName = this.file.name.replace(/\.[^/.]+$/, "");
|
||||
const originalExtension = this.file.name.split(".").pop()!;
|
||||
return name
|
||||
.replace(/%datetime%/g, iso)
|
||||
.replace(/%date%/g, date)
|
||||
.replace(/%time%/g, time)
|
||||
.replace(/%unix%/g, unix)
|
||||
.replace(/%name%/g, baseName)
|
||||
.replace(/%extension%/g, originalExtension);
|
||||
const filename = `${formatFilename(filenameFormat, this.file)}${to}`;
|
||||
const resultFile = this.result.file;
|
||||
|
||||
const filePicker = window as Window & {
|
||||
showSaveFilePicker?: (options?: {
|
||||
suggestedName?: string;
|
||||
types?: Array<{
|
||||
description?: string;
|
||||
accept: Record<string, string[]>;
|
||||
}>;
|
||||
}) => Promise<FileSystemFileHandle>;
|
||||
};
|
||||
|
||||
const filename = `${format(filenameFormat)}${to}`;
|
||||
const diskStreamSupported =
|
||||
typeof filePicker.showSaveFilePicker === "function";
|
||||
const shouldDiskStream =
|
||||
diskStreamSupported && resultFile.size >= LARGE_FILE;
|
||||
|
||||
// larger files (>2gb) requires cache API
|
||||
// blob constructor can't use arraybuffer above 2gb
|
||||
const useCacheApi = this.result.file.size > MAX_BLOB_SIZE_LIMIT;
|
||||
let blob: string;
|
||||
if (shouldDiskStream) {
|
||||
// use the File System Access API to directly stream to disk, so we can actually save larger files
|
||||
try {
|
||||
const ext = to.slice(1);
|
||||
const handle = await filePicker.showSaveFilePicker!({
|
||||
suggestedName: filename,
|
||||
types: [
|
||||
{
|
||||
description: "The VERT converted file",
|
||||
accept: { "application/octet-stream": [`.${ext}`] },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
if (useCacheApi) {
|
||||
const cache = await caches.open("vert-downloads");
|
||||
const cacheKey = `vert-download-${Date.now()}-${filename}`;
|
||||
|
||||
const response = new Response(this.result.file.stream(), {
|
||||
headers: {
|
||||
"Content-Type": "application/octet-stream",
|
||||
"Content-Length": this.result.file.size.toString(),
|
||||
},
|
||||
});
|
||||
|
||||
await cache.put(cacheKey, response);
|
||||
|
||||
const cachedResponse = await cache.match(cacheKey);
|
||||
if (!cachedResponse)
|
||||
throw new Error("Failed to cache file for download");
|
||||
|
||||
const cachedBlob = await cachedResponse.blob();
|
||||
blob = URL.createObjectURL(cachedBlob);
|
||||
|
||||
setTimeout(() => {
|
||||
cache.delete(cacheKey);
|
||||
}, 30000);
|
||||
} else {
|
||||
blob = URL.createObjectURL(
|
||||
new Blob([await this.result.file.arrayBuffer()], {
|
||||
type: "application/octet-stream",
|
||||
}),
|
||||
);
|
||||
const writable = await handle.createWritable();
|
||||
await resultFile.stream().pipeTo(writable);
|
||||
this.blobUrl = undefined;
|
||||
return;
|
||||
} catch (err) {
|
||||
const casted = err as DOMException;
|
||||
if (casted?.name === "AbortError") return;
|
||||
log(
|
||||
["file", "download"],
|
||||
`disk-streaming download failed, falling back to blob URL: ${err}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// fallback to blob URL download for smaller files or if the File System Access API isn't supported
|
||||
const blob = URL.createObjectURL(resultFile);
|
||||
|
||||
// download
|
||||
const a = document.createElement("a");
|
||||
a.href = blob;
|
||||
|
|
|
|||
|
|
@ -28,7 +28,10 @@ export async function extractZip(file: File): Promise<ZipEntry[]> {
|
|||
data: new Uint8Array(data),
|
||||
}));
|
||||
|
||||
log(["zip"], `extracted ${entries.length} entries from ${file.name}`);
|
||||
log(
|
||||
["zip"],
|
||||
`extracted ${entries.length} entries from ${file.name}`,
|
||||
);
|
||||
resolve(entries);
|
||||
});
|
||||
});
|
||||
|
|
@ -47,3 +50,27 @@ export function ignoreEntry(filename: string): boolean {
|
|||
filename.endsWith("/")
|
||||
);
|
||||
}
|
||||
|
||||
export function formatFilename(format: string, file: File | string) {
|
||||
const now = new Date();
|
||||
const iso = now.toISOString();
|
||||
const date = iso.split("T")[0];
|
||||
const time = iso.split("T")[1].split(".")[0].replace(/:/g, "-");
|
||||
const unix = now.getTime().toString();
|
||||
const baseName =
|
||||
typeof file === "string"
|
||||
? file.replace(/\.[^/.]+$/, "")
|
||||
: file.name.replace(/\.[^/.]+$/, "");
|
||||
const originalExtension =
|
||||
typeof file === "string"
|
||||
? file.split(".").pop()!
|
||||
: file.name.split(".").pop()!;
|
||||
|
||||
return format
|
||||
.replace(/%datetime%/g, iso)
|
||||
.replace(/%date%/g, date)
|
||||
.replace(/%time%/g, time)
|
||||
.replace(/%unix%/g, unix)
|
||||
.replace(/%name%/g, baseName)
|
||||
.replace(/%extension%/g, originalExtension);
|
||||
}
|
||||
Loading…
Reference in New Issue