Merge branch 'nullptr/experimental-audio-to-video'

src/lib/converters/converter.svelte.ts

@@ -5,8 +5,9 @@ export class FormatInfo {
 
 	constructor(
 		name: string,
-		public fromSupported: boolean,
-		public toSupported: boolean,
+		public fromSupported = true,
+		public toSupported = true,
+		public isNative = true,
 	) {
 		this.name = name;
 		if (!this.name.startsWith(".")) {

src/lib/converters/ffmpeg.svelte.ts

@@ -5,6 +5,8 @@ import { browser } from "$app/environment";
 import { error, log } from "$lib/logger";
 import { addToast } from "$lib/store/ToastProvider";
 
+const videoFormats = ["mp4", "mkv", "avi", "mov", "webm"];
+
 export class FFmpegConverter extends Converter {
 	private ffmpeg: FFmpeg = null!;
 	public name = "ffmpeg";
@@ -17,10 +19,11 @@ export class FFmpegConverter extends Converter {
 		new FormatInfo("ogg", true, true),
 		new FormatInfo("aac", true, true),
 		new FormatInfo("m4a", true, true),
+		...videoFormats.map((f) => new FormatInfo(f, true, true, false)),
 		new FormatInfo("wma", true, true),
 		new FormatInfo("amr", true, true),
 		new FormatInfo("ac3", true, true),
-		new FormatInfo("alac", true, false),
+		new FormatInfo("alac", true, true),
 		new FormatInfo("aiff", true, true),
 	];
 
@@ -57,6 +60,17 @@ export class FFmpegConverter extends Converter {
 		ffmpeg.on("progress", (progress) => {
 			input.progress = progress.progress * 100;
 		});
+		ffmpeg.on("log", (l) => {
+			log(["converters", this.name], l.message);
+
+			if (l.message.includes("Stream map '0:a:0' matches no streams.")) {
+				error(
+					["converters", this.name],
+					`No audio stream found in ${input.name}.`,
+				);
+				addToast("error", `No audio stream found in ${input.name}.`);
+			}
+		});
 		const baseURL =
 			"https://cdn.jsdelivr.net/npm/@ffmpeg/core@0.12.6/dist/esm";
 		await ffmpeg.load({
@@ -69,7 +83,37 @@ export class FFmpegConverter extends Converter {
 			["converters", this.name],
 			`wrote ${input.name} to ffmpeg virtual fs`,
 		);
-		await ffmpeg.exec(["-i", "input", "output" + to]);
+		if (videoFormats.includes(input.from.slice(1))) {
+			// create an audio track from the video
+			await ffmpeg.exec(["-i", "input", "-map", "0:a:0", "output" + to]);
+		} else if (videoFormats.includes(to.slice(1))) {
+			// nab the album art
+			await ffmpeg.exec([
+				"-i",
+				"input",
+				"-an",
+				"-vcodec",
+				"copy",
+				"cover.png",
+			]);
+			const cmd = [
+				"-i",
+				"input",
+				"-i",
+				"cover.png",
+				"-loop",
+				"1",
+				"-pix_fmt",
+				"yuv420p",
+				...toArgs(to),
+				"output" + to,
+			];
+			console.log(cmd);
+			await ffmpeg.exec(cmd);
+		} else {
+			await ffmpeg.exec(["-i", "input", "output" + to]);
+		}
+
 		log(["converters", this.name], `executed ffmpeg command`);
 		const output = (await ffmpeg.readFile(
 			"output" + to,
@@ -82,3 +126,49 @@ export class FFmpegConverter extends Converter {
 		return new VertFile(new File([output], input.name), to);
 	}
 }
+
+// and here i was, thinking i'd be done with ffmpeg after finishing vertd
+// but OH NO we just HAD to have someone suggest to allow album art video generation.
+//
+// i hate you SO much.
+// - love, maddie
+const toArgs = (ext: string): string[] => {
+	const encoder = getEncoder(ext);
+	const args = ["-c:v", encoder];
+	switch (encoder) {
+		case "libx264": {
+			args.push(
+				"-preset",
+				"ultrafast",
+				"-crf",
+				"18",
+				"-tune",
+				"stillimage",
+				"-c:a",
+				"aac",
+			);
+			break;
+		}
+
+		case "libvpx": {
+			args.push("-c:v", "libvpx-vp9", "-c:a", "libvorbis");
+			break;
+		}
+	}
+
+	return args;
+};
+
+const getEncoder = (ext: string): string => {
+	switch (ext) {
+		case ".mkv":
+		case ".mp4":
+		case ".avi":
+		case ".mov":
+			return "libx264";
+		case ".webm":
+			return "libvpx";
+		default:
+			return "copy";
+	}
+};

src/lib/converters/index.ts

@@ -1,4 +1,5 @@
 import type { Categories } from "$lib/types";
+import type { Converter } from "./converter.svelte";
 import { FFmpegConverter } from "./ffmpeg.svelte";
 import { PandocConverter } from "./pandoc.svelte";
 import { VertdConverter } from "./vertd.svelte";
@@ -47,3 +48,14 @@ categories.docs.formats =
 		.find((c) => c.name === "pandoc")
 		?.formatStrings((f) => f.toSupported)
 		.filter((f) => f !== ".pdf") || [];
+export const byNative = (format: string) => {
+	return (a: Converter, b: Converter) => {
+		const aFormat = a.supportedFormats.find((f) => f.name === format);
+		const bFormat = b.supportedFormats.find((f) => f.name === format);
+
+		if (aFormat && bFormat) {
+			return aFormat.isNative ? -1 : 1;
+		}
+		return 0;
+	};
+};

src/lib/converters/vips.svelte.ts

@@ -17,11 +17,11 @@ export class VipsConverter extends Converter {
 	public ready = $state(false);
 
 	public supportedFormats = [
-		new FormatInfo("png", true, true),
-		new FormatInfo("jpeg", true, true),
-		new FormatInfo("jpg", true, true),
-		new FormatInfo("webp", true, true),
-		new FormatInfo("gif", true, true),
+		new FormatInfo("png"),
+		new FormatInfo("jpeg"),
+		new FormatInfo("jpg"),
+		new FormatInfo("webp"),
+		new FormatInfo("gif"),
 		new FormatInfo("heic", true, false),
 		new FormatInfo("ico", true, false),
 		new FormatInfo("bmp", true, false),
@@ -30,8 +30,8 @@ export class VipsConverter extends Converter {
 		new FormatInfo("icns", true, false),
 		new FormatInfo("nef", true, false),
 		new FormatInfo("cr2", true, false),
-		new FormatInfo("hdr", true, true),
-		new FormatInfo("jpe", true, true),
+		new FormatInfo("hdr"),
+		new FormatInfo("jpe"),
 		new FormatInfo("dng", true, false),
 		new FormatInfo("mat", true, true),
 		new FormatInfo("pbm", true, true),
@@ -40,10 +40,10 @@ export class VipsConverter extends Converter {
 		new FormatInfo("pnm", true, true),
 		new FormatInfo("ppm", false, true),
 		new FormatInfo("raw", false, true),
-		new FormatInfo("tif", true, true),
-		new FormatInfo("tiff", true, true),
-		new FormatInfo("jfif", true, true),
-		new FormatInfo("avif", true, true),
+		new FormatInfo("tif"),
+		new FormatInfo("tiff"),
+		new FormatInfo("jfif"),
+		new FormatInfo("avif"),
 	];
 
 	public readonly reportsProgress = false;

src/lib/store/index.svelte.ts

@@ -1,5 +1,5 @@
 import { browser } from "$app/environment";
-import { converters } from "$lib/converters";
+import { byNative, converters } from "$lib/converters";
 import { error, log } from "$lib/logger";
 import { VertFile } from "$lib/types";
 import { parseBlob, selectCover } from "music-metadata";
@@ -32,11 +32,13 @@ class Files {
 		this.thumbnailQueue.add(async () => {
 			const isAudio = converters
 				.find((c) => c.name === "ffmpeg")
-				?.formatStrings()
+				?.supportedFormats.filter((f) => f.isNative)
+				.map((f) => f.name)
 				?.includes(file.from.toLowerCase());
 			const isVideo = converters
 				.find((c) => c.name === "vertd")
-				?.formatStrings()
+				?.supportedFormats.filter((f) => f.isNative)
+				.map((f) => f.name)
 				?.includes(file.from.toLowerCase());
 
 			try {
@@ -120,11 +122,11 @@ class Files {
 				log(["files"], `no extension found for ${file.name}`);
 				return;
 			}
-			const converter = converters.find((c) =>
-				c
-					.formatStrings()
-					.includes(format || ".somenonexistentextension"),
-			);
+			const converter = converters
+				.sort(byNative(format))
+				.find((converter) =>
+					converter.formatStrings().includes(format),
+				);
 			if (!converter) {
 				log(["files"], `no converter found for ${file.name}`);
 				this.files.push(new VertFile(file, format));

src/lib/types/file.svelte.ts

@@ -1,4 +1,4 @@
-import { converters } from "$lib/converters";
+import { byNative, converters } from "$lib/converters";
 import type { Converter } from "$lib/converters/converter.svelte";
 import { error } from "$lib/logger";
 import { addToast } from "$lib/store/ToastProvider";
@@ -27,18 +27,35 @@ export class VertFile {
 	public converters: Converter[] = [];
 
 	public findConverters(supportedFormats: string[] = [this.from]) {
-		const converter = this.converters.filter((converter) =>
-			converter.formatStrings().map((f) => supportedFormats.includes(f)),
-		);
+		const converter = this.converters
+			.filter((converter) =>
+				converter
+					.formatStrings()
+					.map((f) => supportedFormats.includes(f)),
+			)
+			.sort(byNative(this.from));
 		return converter;
 	}
 
 	public findConverter() {
-		const converter = this.converters.find(
-			(converter) =>
-				converter.formatStrings().includes(this.from) &&
-				converter.formatStrings().includes(this.to),
-		);
+		const converter = this.converters.find((converter) => {
+			if (
+				!converter.formatStrings().includes(this.from) ||
+				!converter.formatStrings().includes(this.to)
+			) {
+				return false;
+			}
+
+			const theirFrom = converter.supportedFormats.find(
+				(f) => f.name === this.from,
+			);
+			const theirTo = converter.supportedFormats.find(
+				(f) => f.name === this.to,
+			);
+			if (!theirFrom || !theirTo) return false;
+			if (!theirFrom.isNative && !theirTo.isNative) return false;
+			return true;
+		});
 		return converter;
 	}
 

src/routes/convert/+page.svelte

@@ -5,7 +5,7 @@
 	import Panel from "$lib/components/visual/Panel.svelte";
 	import ProgressBar from "$lib/components/visual/ProgressBar.svelte";
 	import Tooltip from "$lib/components/visual/Tooltip.svelte";
-	import { categories, converters } from "$lib/converters";
+	import { categories, converters, byNative } from "$lib/converters";
 	import {
 		effects,
 		files,
@@ -60,21 +60,30 @@
 	$effect(() => {
 		// Set gradient color depending on the file types
 		// TODO: if more file types added, add a "fileType" property to the file object
-		const allAudio = files.files.every(
-			(file) => file.findConverter()?.name === "ffmpeg",
-		);
-		const allImages = files.files.every(
-			(file) =>
-				file.findConverter()?.name !== "ffmpeg" &&
-				file.findConverter()?.name !== "vertd",
-		);
-		const allVideos = files.files.every(
-			(file) => file.findConverter()?.name === "vertd",
-		);
-
-		const allDocuments = files.files.every(
-			(file) => file.findConverter()?.name === "pandoc",
-		);
+		const allAudio = files.files.every((file) => {
+			const converter = file
+				.findConverters()
+				.sort(byNative(file.from))[0];
+			return converter?.name === "ffmpeg";
+		});
+		const allImages = files.files.every((file) => {
+			const converter = file
+				.findConverters()
+				.sort(byNative(file.from))[0];
+			return converter?.name === "libvips";
+		});
+		const allVideos = files.files.every((file) => {
+			const converter = file
+				.findConverters()
+				.sort(byNative(file.from))[0];
+			return converter?.name === "vertd";
+		});
+		const allDocuments = files.files.every((file) => {
+			const converter = file
+				.findConverters()
+				.sort(byNative(file.from))[0];
+			return converter?.name === "pandoc";
+		});
 
 		if (files.files.length === 1 && files.files[0].blobUrl && !allVideos) {
 			showGradient.set(false);
@@ -84,7 +93,7 @@
 
 		if (
 			files.files.length === 0 ||
-			(!allAudio && !allImages && !allVideos)
+			(!allAudio && !allImages && !allVideos && !allDocuments)
 		) {
 			gradientColor.set("");
 		} else {
@@ -112,19 +121,23 @@
 	)}
 	{@const isAudio = converters
 		.find((c) => c.name === "ffmpeg")
-		?.formatStrings((f) => f.fromSupported)
+		?.supportedFormats.filter((f) => f.isNative)
+		.map((f) => f.name)
 		.includes(file.from)}
 	{@const isVideo = converters
 		.find((c) => c.name === "vertd")
-		?.formatStrings((f) => f.fromSupported)
+		?.supportedFormats.filter((f) => f.isNative)
+		.map((f) => f.name)
 		.includes(file.from)}
 	{@const isImage = converters
 		.find((c) => c.name === "libvips")
-		?.formatStrings((f) => f.fromSupported)
+		?.supportedFormats.filter((f) => f.isNative)
+		.map((f) => f.name)
 		.includes(file.from)}
 	{@const isDocument = converters
 		.find((c) => c.name === "pandoc")
-		?.formatStrings((f) => f.fromSupported)
+		?.supportedFormats.filter((f) => f.isNative)
+		.map((f) => f.name)
 		.includes(file.from)}
 	<Panel class="p-5 flex flex-col min-w-0 gap-4 relative">
 		<div class="flex-shrink-0 h-8 w-full flex items-center gap-2">