fix: surface unsupported video codec on import, fix Input leaks
This commit is contained in:
Maze Winther
parent
cceb835a84
commit
0dffefdd59
|
|
@ -1,43 +1,81 @@
|
|||
import { Input, ALL_FORMATS, BlobSource } from "mediabunny";
|
||||
import {
|
||||
Input,
|
||||
ALL_FORMATS,
|
||||
BlobSource,
|
||||
VideoSampleSink,
|
||||
type VideoCodec,
|
||||
} from "mediabunny";
|
||||
import { createTimelineAudioBuffer } from "@/media/audio";
|
||||
import type { SceneTracks } from "@/timeline";
|
||||
import type { MediaAsset } from "@/media/types";
|
||||
import { TICKS_PER_SECOND } from "@/wasm";
|
||||
import { renderThumbnailDataUrl } from "./thumbnail";
|
||||
|
||||
export async function getVideoInfo({
|
||||
videoFile,
|
||||
}: {
|
||||
videoFile: File;
|
||||
}): Promise<{
|
||||
export type VideoFileData = {
|
||||
duration: number;
|
||||
width: number;
|
||||
height: number;
|
||||
fps: number;
|
||||
hasAudio: boolean;
|
||||
}> {
|
||||
codec: VideoCodec | null;
|
||||
canDecode: boolean;
|
||||
thumbnailUrl: string | null;
|
||||
};
|
||||
|
||||
export async function readVideoFile({
|
||||
file,
|
||||
}: {
|
||||
file: File;
|
||||
}): Promise<VideoFileData> {
|
||||
const input = new Input({
|
||||
source: new BlobSource(videoFile),
|
||||
source: new BlobSource(file),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const duration = await input.computeDuration();
|
||||
const videoTrack = await input.getPrimaryVideoTrack();
|
||||
try {
|
||||
const duration = await input.computeDuration();
|
||||
const videoTrack = await input.getPrimaryVideoTrack();
|
||||
|
||||
if (!videoTrack) {
|
||||
throw new Error("No video track found in the file");
|
||||
if (!videoTrack) {
|
||||
throw new Error("No video track found in the file");
|
||||
}
|
||||
|
||||
const canDecode = await videoTrack.canDecode();
|
||||
const packetStats = await videoTrack.computePacketStats(100);
|
||||
const audioTrack = await input.getPrimaryAudioTrack();
|
||||
|
||||
let thumbnailUrl: string | null = null;
|
||||
if (canDecode) {
|
||||
const sink = new VideoSampleSink(videoTrack);
|
||||
const frame = await sink.getSample(1);
|
||||
if (frame) {
|
||||
try {
|
||||
thumbnailUrl = renderThumbnailDataUrl({
|
||||
width: videoTrack.displayWidth,
|
||||
height: videoTrack.displayHeight,
|
||||
draw: ({ context, width, height }) => {
|
||||
frame.draw(context, 0, 0, width, height);
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
frame.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
duration,
|
||||
width: videoTrack.displayWidth,
|
||||
height: videoTrack.displayHeight,
|
||||
fps: packetStats.averagePacketRate,
|
||||
hasAudio: audioTrack !== null,
|
||||
codec: videoTrack.codec,
|
||||
canDecode,
|
||||
thumbnailUrl,
|
||||
};
|
||||
} finally {
|
||||
input.dispose();
|
||||
}
|
||||
|
||||
const packetStats = await videoTrack.computePacketStats(100);
|
||||
const fps = packetStats.averagePacketRate;
|
||||
const audioTrack = await input.getPrimaryAudioTrack();
|
||||
|
||||
return {
|
||||
duration,
|
||||
width: videoTrack.displayWidth,
|
||||
height: videoTrack.displayHeight,
|
||||
fps,
|
||||
hasAudio: audioTrack !== null,
|
||||
};
|
||||
}
|
||||
|
||||
const SAMPLE_RATE = 44100;
|
||||
|
|
|
|||
|
|
@ -1,15 +1,25 @@
|
|||
import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
|
||||
import { toast } from "sonner";
|
||||
import { getMediaTypeFromFile } from "@/media/media-utils";
|
||||
import { formatStorageBytes } from "@/services/storage/quota";
|
||||
import { storageService } from "@/services/storage/service";
|
||||
import type { MediaAsset } from "@/media/types";
|
||||
import { getVideoInfo } from "./mediabunny";
|
||||
import { readVideoFile } from "./mediabunny";
|
||||
import type { VideoFileData } from "./mediabunny";
|
||||
import { renderThumbnailDataUrl } from "./thumbnail";
|
||||
|
||||
export interface ProcessedMediaAsset extends Omit<MediaAsset, "id"> {}
|
||||
|
||||
const THUMBNAIL_MAX_WIDTH = 1280;
|
||||
const THUMBNAIL_MAX_HEIGHT = 720;
|
||||
const getUnsupportedVideoDescription = ({
|
||||
codec,
|
||||
}: {
|
||||
codec: VideoFileData["codec"];
|
||||
}): string => {
|
||||
const codecLabel = codec ? codec.toUpperCase() : "this video codec";
|
||||
|
||||
return codec === "hevc"
|
||||
? `${codecLabel} cannot be decoded in this browser, so this clip may not preview correctly. Convert it to H.264 MP4 or try importing it in Safari.`
|
||||
: `${codecLabel} cannot be decoded in this browser, so this clip may not preview correctly. Convert it to H.264 MP4 and reimport it.`;
|
||||
};
|
||||
|
||||
const getStorageLimitDescription = ({
|
||||
fileSize,
|
||||
|
|
@ -29,103 +39,6 @@ const getStorageLimitDescription = ({
|
|||
})} is safely available in browser storage.`;
|
||||
};
|
||||
|
||||
const getThumbnailSize = ({
|
||||
width,
|
||||
height,
|
||||
}: {
|
||||
width: number;
|
||||
height: number;
|
||||
}): { width: number; height: number } => {
|
||||
const aspectRatio = width / height;
|
||||
let targetWidth = width;
|
||||
let targetHeight = height;
|
||||
|
||||
if (targetWidth > THUMBNAIL_MAX_WIDTH) {
|
||||
targetWidth = THUMBNAIL_MAX_WIDTH;
|
||||
targetHeight = Math.round(targetWidth / aspectRatio);
|
||||
}
|
||||
if (targetHeight > THUMBNAIL_MAX_HEIGHT) {
|
||||
targetHeight = THUMBNAIL_MAX_HEIGHT;
|
||||
targetWidth = Math.round(targetHeight * aspectRatio);
|
||||
}
|
||||
|
||||
return { width: targetWidth, height: targetHeight };
|
||||
};
|
||||
|
||||
const renderToThumbnailDataUrl = ({
|
||||
width,
|
||||
height,
|
||||
draw,
|
||||
}: {
|
||||
width: number;
|
||||
height: number;
|
||||
draw: ({
|
||||
context,
|
||||
width,
|
||||
height,
|
||||
}: {
|
||||
context: CanvasRenderingContext2D;
|
||||
width: number;
|
||||
height: number;
|
||||
}) => void;
|
||||
}): string => {
|
||||
const size = getThumbnailSize({ width, height });
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = size.width;
|
||||
canvas.height = size.height;
|
||||
const context = canvas.getContext("2d");
|
||||
|
||||
if (!context) {
|
||||
throw new Error("Could not get canvas context");
|
||||
}
|
||||
|
||||
draw({ context, width: size.width, height: size.height });
|
||||
return canvas.toDataURL("image/jpeg", 0.8);
|
||||
};
|
||||
|
||||
async function generateThumbnail({
|
||||
videoFile,
|
||||
timeInSeconds,
|
||||
}: {
|
||||
videoFile: File;
|
||||
timeInSeconds: number;
|
||||
}): Promise<string> {
|
||||
const input = new Input({
|
||||
source: new BlobSource(videoFile),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const videoTrack = await input.getPrimaryVideoTrack();
|
||||
if (!videoTrack) {
|
||||
throw new Error("No video track found in the file");
|
||||
}
|
||||
|
||||
const canDecode = await videoTrack.canDecode();
|
||||
if (!canDecode) {
|
||||
throw new Error("Video codec not supported for decoding");
|
||||
}
|
||||
|
||||
const sink = new VideoSampleSink(videoTrack);
|
||||
|
||||
const frame = await sink.getSample(timeInSeconds);
|
||||
|
||||
if (!frame) {
|
||||
throw new Error("Could not get frame at specified time");
|
||||
}
|
||||
|
||||
try {
|
||||
return renderToThumbnailDataUrl({
|
||||
width: videoTrack.displayWidth,
|
||||
height: videoTrack.displayHeight,
|
||||
draw: ({ context, width, height }) => {
|
||||
frame.draw(context, 0, 0, width, height);
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
frame.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function generateImageThumbnail({
|
||||
imageFile,
|
||||
}: {
|
||||
|
|
@ -137,7 +50,7 @@ async function generateImageThumbnail({
|
|||
|
||||
image.addEventListener("load", () => {
|
||||
try {
|
||||
const thumbnailUrl = renderToThumbnailDataUrl({
|
||||
const thumbnailUrl = renderThumbnailDataUrl({
|
||||
width: image.naturalWidth,
|
||||
height: image.naturalHeight,
|
||||
draw: ({ context, width, height }) => {
|
||||
|
|
@ -220,24 +133,34 @@ export async function processMediaAssets({
|
|||
height = result.height;
|
||||
} else if (fileType === "video") {
|
||||
try {
|
||||
const videoInfo = await getVideoInfo({ videoFile: file });
|
||||
duration = videoInfo.duration;
|
||||
width = videoInfo.width;
|
||||
height = videoInfo.height;
|
||||
fps = Number.isFinite(videoInfo.fps)
|
||||
? Math.round(videoInfo.fps)
|
||||
const videoData = await readVideoFile({ file });
|
||||
duration = videoData.duration;
|
||||
width = videoData.width;
|
||||
height = videoData.height;
|
||||
fps = Number.isFinite(videoData.fps)
|
||||
? Math.round(videoData.fps)
|
||||
: undefined;
|
||||
hasAudio = videoInfo.hasAudio;
|
||||
hasAudio = videoData.hasAudio;
|
||||
thumbnailUrl = videoData.thumbnailUrl ?? undefined;
|
||||
|
||||
thumbnailUrl = await generateThumbnail({
|
||||
videoFile: file,
|
||||
timeInSeconds: 1,
|
||||
});
|
||||
if (!videoData.canDecode) {
|
||||
toast.error(`Can't preview ${file.name}`, {
|
||||
description: getUnsupportedVideoDescription({
|
||||
codec: videoData.codec,
|
||||
}),
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("Video processing failed", error);
|
||||
const message =
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: "Could not process video";
|
||||
|
||||
toast.error(`Couldn't process ${file.name}`, {
|
||||
description: message,
|
||||
});
|
||||
}
|
||||
} else if (fileType === "audio") {
|
||||
// For audio, we don't set width/height/fps (they'll be undefined)
|
||||
duration = await getMediaDuration({ file });
|
||||
}
|
||||
|
||||
|
|
@ -264,7 +187,7 @@ export async function processMediaAssets({
|
|||
} catch (error) {
|
||||
console.error("Error processing file:", file.name, error);
|
||||
toast.error(`Failed to process ${file.name}`);
|
||||
URL.revokeObjectURL(url); // Clean up on error
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
const THUMBNAIL_MAX_WIDTH = 1280;
|
||||
const THUMBNAIL_MAX_HEIGHT = 720;
|
||||
|
||||
export function thumbnailSize({
|
||||
width,
|
||||
height,
|
||||
}: {
|
||||
width: number;
|
||||
height: number;
|
||||
}): { width: number; height: number } {
|
||||
const aspectRatio = width / height;
|
||||
let targetWidth = width;
|
||||
let targetHeight = height;
|
||||
|
||||
if (targetWidth > THUMBNAIL_MAX_WIDTH) {
|
||||
targetWidth = THUMBNAIL_MAX_WIDTH;
|
||||
targetHeight = Math.round(targetWidth / aspectRatio);
|
||||
}
|
||||
if (targetHeight > THUMBNAIL_MAX_HEIGHT) {
|
||||
targetHeight = THUMBNAIL_MAX_HEIGHT;
|
||||
targetWidth = Math.round(targetHeight * aspectRatio);
|
||||
}
|
||||
|
||||
return { width: targetWidth, height: targetHeight };
|
||||
}
|
||||
|
||||
export function renderThumbnailDataUrl({
|
||||
width,
|
||||
height,
|
||||
draw,
|
||||
}: {
|
||||
width: number;
|
||||
height: number;
|
||||
draw: ({
|
||||
context,
|
||||
width,
|
||||
height,
|
||||
}: {
|
||||
context: CanvasRenderingContext2D;
|
||||
width: number;
|
||||
height: number;
|
||||
}) => void;
|
||||
}): string {
|
||||
const size = thumbnailSize({ width, height });
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = size.width;
|
||||
canvas.height = size.height;
|
||||
const context = canvas.getContext("2d");
|
||||
|
||||
if (!context) {
|
||||
throw new Error("Could not get canvas context");
|
||||
}
|
||||
|
||||
draw({ context, width: size.width, height: size.height });
|
||||
return canvas.toDataURL("image/jpeg", 0.8);
|
||||
}
|
||||
|
|
@ -7,6 +7,7 @@ import {
|
|||
} from "mediabunny";
|
||||
|
||||
interface VideoSinkData {
|
||||
input: Input;
|
||||
sink: CanvasSink;
|
||||
iterator: AsyncGenerator<WrappedCanvas, void, unknown> | null;
|
||||
currentFrame: WrappedCanvas | null;
|
||||
|
|
@ -262,12 +263,12 @@ export class VideoCache {
|
|||
mediaId: string;
|
||||
file: File;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
const input = new Input({
|
||||
source: new BlobSource(file),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
const input = new Input({
|
||||
source: new BlobSource(file),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
try {
|
||||
const videoTrack = await input.getPrimaryVideoTrack();
|
||||
if (!videoTrack) {
|
||||
throw new Error("No video track found");
|
||||
|
|
@ -284,6 +285,7 @@ export class VideoCache {
|
|||
});
|
||||
|
||||
this.sinks.set(mediaId, {
|
||||
input,
|
||||
sink,
|
||||
iterator: null,
|
||||
currentFrame: null,
|
||||
|
|
@ -293,6 +295,7 @@ export class VideoCache {
|
|||
prefetchPromise: null,
|
||||
});
|
||||
} catch (error) {
|
||||
input.dispose();
|
||||
console.error(`Failed to initialize video sink for ${mediaId}:`, error);
|
||||
throw error;
|
||||
}
|
||||
|
|
@ -305,6 +308,7 @@ export class VideoCache {
|
|||
void sinkData.iterator.return();
|
||||
}
|
||||
|
||||
sinkData.input.dispose();
|
||||
this.sinks.delete(mediaId);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue