feat: mediabunny + preview canvas refactor

This commit is contained in:
Maze Winther 2025-08-23 11:00:59 +02:00
parent 7756ed0d80
commit a4f2273e25
7 changed files with 593 additions and 462 deletions

View File

@ -3,7 +3,7 @@ import { PropertyGroup } from "../../properties-panel/property-item";
import { BaseView } from "./base-view";
import { Language, LanguageSelect } from "@/components/language-select";
import { useState, useRef, useEffect } from "react";
import { extractTimelineAudio } from "@/lib/ffmpeg-utils";
import { extractTimelineAudio } from "@/lib/mediabunny-utils";
import { encryptWithRandomKey, arrayBufferToBase64 } from "@/lib/zk-encryption";
import { useTimelineStore } from "@/stores/timeline-store";
import { Loader2, Shield, Trash2, Upload } from "lucide-react";

View File

@ -5,16 +5,19 @@ import { TimelineElement, TimelineTrack } from "@/types/timeline";
import { useMediaStore, type MediaItem } from "@/stores/media-store";
import { usePlaybackStore } from "@/stores/playback-store";
import { useEditorStore } from "@/stores/editor-store";
import { VideoPlayer } from "@/components/ui/video-player";
import { AudioPlayer } from "@/components/ui/audio-player";
import { Button } from "@/components/ui/button";
import { Play, Pause, Expand, SkipBack, SkipForward } from "lucide-react";
import { useState, useRef, useEffect, useCallback } from "react";
import { VideoSampleSink } from "mediabunny";
import { renderTimelineFrame } from "@/lib/timeline-renderer";
import { cn } from "@/lib/utils";
import { formatTimeCode } from "@/lib/time";
import { EditableTimecode } from "@/components/ui/editable-timecode";
import { FONT_CLASS_MAP } from "@/lib/font-config";
import { DEFAULT_CANVAS_SIZE, DEFAULT_FPS, useProjectStore } from "@/stores/project-store";
import {
DEFAULT_CANVAS_SIZE,
DEFAULT_FPS,
useProjectStore,
} from "@/stores/project-store";
import { TextElementDragState } from "@/types/editor";
import {
Popover,
@ -37,8 +40,20 @@ export function PreviewPanel() {
const { tracks, getTotalDuration, updateTextElement } = useTimelineStore();
const { mediaItems } = useMediaStore();
const { currentTime, toggle, setCurrentTime } = usePlaybackStore();
const { isPlaying, volume, muted } = usePlaybackStore();
const { activeProject } = useProjectStore();
const previewRef = useRef<HTMLDivElement>(null);
const canvasRef = useRef<HTMLCanvasElement>(null);
const lastFrameTimeRef = useRef(0);
const renderSeqRef = useRef(0);
const offscreenCanvasRef = useRef<OffscreenCanvas | HTMLCanvasElement | null>(
null
);
const audioContextRef = useRef<AudioContext | null>(null);
const audioGainRef = useRef<GainNode | null>(null);
const audioBuffersRef = useRef<Map<string, AudioBuffer>>(new Map());
const playingSourcesRef = useRef<Set<AudioBufferSourceNode>>(new Set());
const containerRef = useRef<HTMLDivElement>(null);
const [previewDimensions, setPreviewDimensions] = useState({
width: 0,
@ -262,6 +277,296 @@ export function PreviewPanel() {
const activeElements = getActiveElements();
// Ensure first frame after mount/seek renders immediately
useEffect(() => {
const onSeek = () => {
lastFrameTimeRef.current = -Infinity;
renderSeqRef.current++;
};
window.addEventListener("playback-seek", onSeek as EventListener);
lastFrameTimeRef.current = -Infinity;
return () => {
window.removeEventListener("playback-seek", onSeek as EventListener);
};
}, []);
// Web Audio: schedule only on play/pause/seek/volume/mute changes
useEffect(() => {
const stopAll = () => {
for (const src of playingSourcesRef.current) {
try {
src.stop();
} catch {}
}
playingSourcesRef.current.clear();
};
type WebAudioWindow = Window & {
AudioContext?: typeof AudioContext;
webkitAudioContext?: typeof AudioContext;
};
const ensureAudioGraph = async () => {
if (!audioContextRef.current) {
const win = window as WebAudioWindow;
const Ctx = win.AudioContext ?? win.webkitAudioContext;
if (!Ctx) return;
audioContextRef.current = new Ctx();
}
if (!audioGainRef.current) {
audioGainRef.current = audioContextRef.current!.createGain();
audioGainRef.current.connect(audioContextRef.current!.destination);
}
if (audioContextRef.current!.state === "suspended") {
try {
await audioContextRef.current!.resume();
} catch {}
}
const gainValue = muted ? 0 : Math.max(0, Math.min(1, volume));
audioGainRef.current!.gain.setValueAtTime(
gainValue,
audioContextRef.current!.currentTime
);
};
const scheduleNow = async () => {
await ensureAudioGraph();
const audioCtx = audioContextRef.current!;
const gain = audioGainRef.current!;
const tracksSnapshot = useTimelineStore.getState().tracks;
const mediaList = useMediaStore.getState().mediaItems;
const idToMedia = new Map(mediaList.map((m) => [m.id, m] as const));
const playbackNow = usePlaybackStore.getState().currentTime;
const audible: Array<{
id: string;
elementStart: number;
trimStart: number;
trimEnd: number;
duration: number;
muted: boolean;
trackMuted: boolean;
}> = [];
const uniqueIds = new Set<string>();
for (const track of tracksSnapshot) {
for (const element of track.elements) {
if (element.type !== "media") continue;
const media = idToMedia.get(element.mediaId);
if (!media || media.type !== "audio") continue;
const visibleDuration =
element.duration - element.trimStart - element.trimEnd;
if (visibleDuration <= 0) continue;
const localTime = playbackNow - element.startTime + element.trimStart;
if (localTime < 0 || localTime >= visibleDuration) continue;
audible.push({
id: media.id,
elementStart: element.startTime,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
duration: element.duration,
muted: !!element.muted,
trackMuted: !!track.muted,
});
uniqueIds.add(media.id);
}
}
if (audible.length === 0) return;
// Decode buffers as needed
const decodePromises: Array<Promise<void>> = [];
for (const id of uniqueIds) {
if (!audioBuffersRef.current.has(id)) {
const mediaItem = idToMedia.get(id);
if (!mediaItem) continue;
const p = (async () => {
const arr = await mediaItem.file.arrayBuffer();
const buf = await audioCtx.decodeAudioData(arr.slice(0));
audioBuffersRef.current.set(id, buf);
})();
decodePromises.push(p);
}
}
await Promise.all(decodePromises);
const startAt = audioCtx.currentTime + 0.02;
for (const entry of audible) {
if (entry.muted || entry.trackMuted) continue;
const buffer = audioBuffersRef.current.get(entry.id);
if (!buffer) continue;
const visibleDuration =
entry.duration - entry.trimStart - entry.trimEnd;
const localTime = Math.max(
0,
playbackNow - entry.elementStart + entry.trimStart
);
const playDuration = Math.max(0, visibleDuration - localTime);
if (playDuration <= 0) continue;
const src = audioCtx.createBufferSource();
src.buffer = buffer;
src.connect(gain);
try {
src.start(startAt, localTime, playDuration);
playingSourcesRef.current.add(src);
} catch {}
}
};
const onSeek = () => {
if (!isPlaying) return;
for (const src of playingSourcesRef.current) {
try {
src.stop();
} catch {}
}
playingSourcesRef.current.clear();
void scheduleNow();
};
// Apply volume/mute changes immediately
void ensureAudioGraph();
// Start/stop on play state changes
for (const src of playingSourcesRef.current) {
try {
src.stop();
} catch {}
}
playingSourcesRef.current.clear();
if (isPlaying) {
void scheduleNow();
}
window.addEventListener("playback-seek", onSeek as EventListener);
return () => {
window.removeEventListener("playback-seek", onSeek as EventListener);
for (const src of playingSourcesRef.current) {
try {
src.stop();
} catch {}
}
playingSourcesRef.current.clear();
};
}, [isPlaying, volume, muted, mediaItems]);
// Canvas: draw current frame for visible elements using offscreen compositing
useEffect(() => {
const draw = async () => {
const canvas = canvasRef.current;
if (!canvas) return;
const mainCtx = canvas.getContext("2d");
if (!mainCtx) return;
// Set canvas internal resolution to avoid blurry scaling
const displayWidth = Math.max(1, Math.floor(previewDimensions.width));
const displayHeight = Math.max(1, Math.floor(previewDimensions.height));
if (canvas.width !== displayWidth || canvas.height !== displayHeight) {
canvas.width = displayWidth;
canvas.height = displayHeight;
}
// Throttle rendering to project FPS during playback only
const fps = activeProject?.fps || DEFAULT_FPS;
const minDelta = 1 / fps;
if (isPlaying) {
if (currentTime - lastFrameTimeRef.current < minDelta) {
return;
}
lastFrameTimeRef.current = currentTime;
}
// Invalidate older async renders when user scrubs rapidly
const mySeq = (renderSeqRef.current += 1);
// Offscreen buffer to avoid flicker (reuse canvas)
if (!offscreenCanvasRef.current) {
const hasOffscreen =
typeof (globalThis as unknown as { OffscreenCanvas?: unknown })
.OffscreenCanvas !== "undefined";
if (hasOffscreen) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
offscreenCanvasRef.current = new (globalThis as any).OffscreenCanvas(
displayWidth,
displayHeight
) as OffscreenCanvas;
} else {
const c = document.createElement("canvas");
c.width = displayWidth;
c.height = displayHeight;
offscreenCanvasRef.current = c;
}
}
// Ensure size matches
if (
offscreenCanvasRef.current &&
(offscreenCanvasRef.current as HTMLCanvasElement).getContext
) {
const c = offscreenCanvasRef.current as HTMLCanvasElement;
if (c.width !== displayWidth || c.height !== displayHeight) {
c.width = displayWidth;
c.height = displayHeight;
}
} else {
const c = offscreenCanvasRef.current as OffscreenCanvas;
// @ts-ignore width/height exist on OffscreenCanvas in modern browsers
if (
(c as unknown as { width: number }).width !== displayWidth ||
(c as unknown as { height: number }).height !== displayHeight
) {
// @ts-ignore
(c as unknown as { width: number }).width = displayWidth;
// @ts-ignore
(c as unknown as { height: number }).height = displayHeight;
}
}
const offscreenCanvas = offscreenCanvasRef.current as
| HTMLCanvasElement
| OffscreenCanvas;
const offCtx = (offscreenCanvas as HTMLCanvasElement).getContext
? (offscreenCanvas as HTMLCanvasElement).getContext("2d")
: (offscreenCanvas as OffscreenCanvas).getContext("2d");
if (!offCtx) return;
await renderTimelineFrame({
ctx: offCtx as CanvasRenderingContext2D,
time: currentTime,
canvasWidth: displayWidth,
canvasHeight: displayHeight,
tracks,
mediaItems,
backgroundColor:
activeProject?.backgroundType === "blur"
? "transparent"
: activeProject?.backgroundColor || "#000000",
useCache: false,
fps: activeProject?.fps || DEFAULT_FPS,
});
// Blit offscreen to visible canvas
mainCtx.clearRect(0, 0, displayWidth, displayHeight);
if ((offscreenCanvas as HTMLCanvasElement).getContext) {
mainCtx.drawImage(offscreenCanvas as HTMLCanvasElement, 0, 0);
} else {
mainCtx.drawImage(
offscreenCanvas as unknown as CanvasImageSource,
0,
0
);
}
};
void draw();
}, [
activeElements,
currentTime,
previewDimensions.width,
previewDimensions.height,
canvasSize.width,
canvasSize.height,
activeProject?.backgroundType,
activeProject?.backgroundColor,
]);
// Get media elements for blur background (video/image only)
const getBlurBackgroundElements = (): ActiveElement[] => {
return activeElements.filter(
@ -275,213 +580,11 @@ export function PreviewPanel() {
const blurBackgroundElements = getBlurBackgroundElements();
// Render blur background layer
const renderBlurBackground = () => {
if (
!activeProject?.backgroundType ||
activeProject.backgroundType !== "blur" ||
blurBackgroundElements.length === 0
) {
return null;
}
// Render blur background layer (handled by canvas now)
const renderBlurBackground = () => null;
// Use the first media element for background (could be enhanced to use primary/focused element)
const backgroundElement = blurBackgroundElements[0];
const { element, mediaItem } = backgroundElement;
if (!mediaItem) return null;
const blurIntensity = activeProject.blurIntensity || 8;
if (mediaItem.type === "video") {
return (
<div
key={`blur-${element.id}`}
className="absolute inset-0 overflow-hidden"
style={{
filter: `blur(${blurIntensity}px)`,
transform: "scale(1.1)", // Slightly zoom to avoid blur edge artifacts
transformOrigin: "center",
}}
>
<VideoPlayer
src={mediaItem.url!}
poster={mediaItem.thumbnailUrl}
clipStartTime={element.startTime}
trimStart={element.trimStart}
trimEnd={element.trimEnd}
clipDuration={element.duration}
className="w-full h-full object-cover"
trackMuted={true}
/>
</div>
);
}
if (mediaItem.type === "image") {
return (
<div
key={`blur-${element.id}`}
className="absolute inset-0 overflow-hidden"
style={{
filter: `blur(${blurIntensity}px)`,
transform: "scale(1.1)", // Slightly zoom to avoid blur edge artifacts
transformOrigin: "center",
}}
>
<img
src={mediaItem.url!}
alt={mediaItem.name}
className="w-full h-full object-cover"
draggable={false}
/>
</div>
);
}
return null;
};
// Render an element
const renderElement = (elementData: ActiveElement, index: number) => {
const { element, mediaItem } = elementData;
// Text elements
if (element.type === "text") {
const fontClassName =
FONT_CLASS_MAP[element.fontFamily as keyof typeof FONT_CLASS_MAP] || "";
const scaleRatio = previewDimensions.width / canvasSize.width;
return (
<div
key={element.id}
className="absolute cursor-grab"
onMouseDown={(e) =>
handleTextMouseDown(e, element, elementData.track.id)
}
style={{
left: `${
50 +
((dragState.isDragging && dragState.elementId === element.id
? dragState.currentX
: element.x) /
canvasSize.width) *
100
}%`,
top: `${
50 +
((dragState.isDragging && dragState.elementId === element.id
? dragState.currentY
: element.y) /
canvasSize.height) *
100
}%`,
transform: `translate(-50%, -50%) rotate(${element.rotation}deg)`,
opacity: element.opacity,
zIndex: 100 + index, // Text elements on top
}}
>
<div
className={fontClassName}
style={{
fontSize: `${element.fontSize * scaleRatio}px`,
color: element.color,
backgroundColor: element.backgroundColor,
textAlign: element.textAlign,
fontWeight: element.fontWeight,
fontStyle: element.fontStyle,
textDecoration: element.textDecoration,
padding: `${4 * scaleRatio}px ${8 * scaleRatio}px`,
borderRadius: `${2 * scaleRatio}px`,
whiteSpace: "nowrap",
// Fallback for system fonts that don't have classes
...(fontClassName === "" && { fontFamily: element.fontFamily }),
}}
>
{element.content}
</div>
</div>
);
}
// Media elements
if (element.type === "media") {
// Test elements
if (!mediaItem || element.mediaId === "test") {
return (
<div
key={element.id}
className="absolute inset-0 bg-linear-to-br from-blue-500/20 to-purple-500/20 flex items-center justify-center"
>
<div className="text-center">
<div className="text-2xl mb-2">🎬</div>
<p className="text-xs text-foreground">{element.name}</p>
</div>
</div>
);
}
// Video elements
if (mediaItem.type === "video") {
return (
<div
key={element.id}
className="absolute inset-0 flex items-center justify-center"
>
<VideoPlayer
src={mediaItem.url!}
poster={mediaItem.thumbnailUrl}
clipStartTime={element.startTime}
trimStart={element.trimStart}
trimEnd={element.trimEnd}
clipDuration={element.duration}
trackMuted={element.muted || elementData.track.muted}
/>
</div>
);
}
// Image elements
if (mediaItem.type === "image") {
return (
<div
key={element.id}
className="absolute inset-0 flex items-center justify-center"
>
<img
src={mediaItem.url!}
alt={mediaItem.name}
className="max-w-full max-h-full object-contain"
draggable={false}
/>
</div>
);
}
// Audio elements (no visual representation)
if (mediaItem.type === "audio") {
return (
<div
key={element.id}
className="absolute inset-0"
style={{ pointerEvents: "none" }}
>
<AudioPlayer
src={mediaItem.url!}
clipStartTime={element.startTime}
trimStart={element.trimStart}
trimEnd={element.trimEnd}
clipDuration={element.duration}
trackMuted={element.muted || elementData.track.muted}
/>
</div>
);
}
}
return null;
};
// Render an element (canvas handles visuals now). Audio playback to be implemented via Web Audio.
const renderElement = (_elementData: ActiveElement) => null;
return (
<>
@ -505,14 +608,23 @@ export function PreviewPanel() {
}}
>
{renderBlurBackground()}
<canvas
ref={canvasRef}
style={{
position: "absolute",
left: 0,
top: 0,
width: previewDimensions.width,
height: previewDimensions.height,
}}
aria-label="Video preview canvas"
/>
{activeElements.length === 0 ? (
<div className="absolute inset-0 flex items-center justify-center text-muted-foreground">
No elements at current time
</div>
) : (
activeElements.map((elementData, index) =>
renderElement(elementData, index)
)
activeElements.map((elementData) => renderElement(elementData))
)}
<LayoutGuideOverlay />
</div>

View File

@ -6,7 +6,7 @@ import {
getImageDimensions,
type MediaItem,
} from "@/stores/media-store";
import { generateThumbnail, getVideoInfo } from "./ffmpeg-utils";
import { generateThumbnail, getVideoInfo } from "./mediabunny-utils";
export interface ProcessedMediaItem extends Omit<MediaItem, "id"> {}
@ -37,33 +37,23 @@ export async function processMediaFiles(
try {
if (fileType === "image") {
// Get image dimensions
const dimensions = await getImageDimensions(file);
width = dimensions.width;
height = dimensions.height;
} else if (fileType === "video") {
try {
// Use FFmpeg for comprehensive video info extraction
const videoInfo = await getVideoInfo(file);
const videoInfo = await getVideoInfo({ videoFile: file });
duration = videoInfo.duration;
width = videoInfo.width;
height = videoInfo.height;
fps = videoInfo.fps;
// Generate thumbnail using FFmpeg
thumbnailUrl = await generateThumbnail(file, 1);
thumbnailUrl = await generateThumbnail({
videoFile: file,
timeInSeconds: 1,
});
} catch (error) {
console.warn(
"FFmpeg processing failed, falling back to basic processing:",
error
);
// Fallback to basic processing
const videoResult = await generateVideoThumbnail(file);
thumbnailUrl = videoResult.thumbnailUrl;
width = videoResult.width;
height = videoResult.height;
duration = await getMediaDuration(file);
// FPS will remain undefined for fallback
console.warn("Video processing failed", error);
}
} else if (fileType === "audio") {
// For audio, we don't set width/height/fps (they'll be undefined)
@ -82,7 +72,6 @@ export async function processMediaFiles(
fps,
});
// Yield back to the event loop to keep the UI responsive
await new Promise((resolve) => setTimeout(resolve, 0));
completed += 1;

View File

@ -1,7 +1,7 @@
import { FFmpeg } from "@ffmpeg/ffmpeg";
import { toBlobURL } from "@ffmpeg/util";
import { useTimelineStore } from "@/stores/timeline-store";
import { useMediaStore } from "@/stores/media-store";
import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
let ffmpeg: FFmpeg | null = null;
@ -14,256 +14,99 @@ export const initFFmpeg = async (): Promise<FFmpeg> => {
return ffmpeg;
};
export const generateThumbnail = async (
videoFile: File,
timeInSeconds = 1
): Promise<string> => {
const ffmpeg = await initFFmpeg();
export async function generateThumbnail({
videoFile,
timeInSeconds,
}: {
videoFile: File;
timeInSeconds: number;
}): Promise<string> {
const input = new Input({
source: new BlobSource(videoFile),
formats: ALL_FORMATS,
});
const inputName = "input.mp4";
const outputName = "thumbnail.jpg";
// Write input file
await ffmpeg.writeFile(
inputName,
new Uint8Array(await videoFile.arrayBuffer())
);
// Generate thumbnail at specific time
await ffmpeg.exec([
"-i",
inputName,
"-ss",
timeInSeconds.toString(),
"-vframes",
"1",
"-vf",
"scale=320:240",
"-q:v",
"2",
outputName,
]);
// Read output file
const data = await ffmpeg.readFile(outputName);
const blob = new Blob([data], { type: "image/jpeg" });
// Cleanup
await ffmpeg.deleteFile(inputName);
await ffmpeg.deleteFile(outputName);
return URL.createObjectURL(blob);
};
export const trimVideo = async (
videoFile: File,
startTime: number,
endTime: number,
onProgress?: (progress: number) => void
): Promise<Blob> => {
const ffmpeg = await initFFmpeg();
const inputName = "input.mp4";
const outputName = "output.mp4";
// Set up progress callback
if (onProgress) {
ffmpeg.on("progress", ({ progress }) => {
onProgress(progress * 100);
});
const videoTrack = await input.getPrimaryVideoTrack();
if (!videoTrack) {
throw new Error("No video track found in the file");
}
// Write input file
await ffmpeg.writeFile(
inputName,
new Uint8Array(await videoFile.arrayBuffer())
);
// Check if we can decode this video
const canDecode = await videoTrack.canDecode();
if (!canDecode) {
throw new Error("Video codec not supported for decoding");
}
const duration = endTime - startTime;
const sink = new VideoSampleSink(videoTrack);
// Trim video
await ffmpeg.exec([
"-i",
inputName,
"-ss",
startTime.toString(),
"-t",
duration.toString(),
"-c",
"copy", // Use stream copy for faster processing
outputName,
]);
const frame = await sink.getSample(timeInSeconds);
// Read output file
const data = await ffmpeg.readFile(outputName);
const blob = new Blob([data], { type: "video/mp4" });
if (!frame) {
throw new Error("Could not get frame at specified time");
}
// Cleanup
await ffmpeg.deleteFile(inputName);
await ffmpeg.deleteFile(outputName);
const canvas = document.createElement("canvas");
canvas.width = 320;
canvas.height = 240;
const ctx = canvas.getContext("2d");
return blob;
};
if (!ctx) {
throw new Error("Could not get canvas context");
}
export const getVideoInfo = async (
videoFile: File
): Promise<{
frame.draw(ctx, 0, 0, 320, 240);
return new Promise((resolve, reject) => {
canvas.toBlob(
(blob) => {
if (blob) {
resolve(URL.createObjectURL(blob));
} else {
reject(new Error("Failed to create thumbnail blob"));
}
},
"image/jpeg",
0.8
);
});
}
export async function getVideoInfo({
videoFile,
}: {
videoFile: File;
}): Promise<{
duration: number;
width: number;
height: number;
fps: number;
}> => {
const ffmpeg = await initFFmpeg();
}> {
const input = new Input({
source: new BlobSource(videoFile),
formats: ALL_FORMATS,
});
const inputName = "input.mp4";
const duration = await input.computeDuration();
const videoTrack = await input.getPrimaryVideoTrack();
// Write input file
await ffmpeg.writeFile(
inputName,
new Uint8Array(await videoFile.arrayBuffer())
);
// Capture FFmpeg stderr output with a one-time listener pattern
let ffmpegOutput = "";
let listening = true;
const listener = (data: string) => {
if (listening) ffmpegOutput += data;
};
ffmpeg.on("log", ({ message }) => listener(message));
// Run ffmpeg to get info (stderr will contain the info)
try {
await ffmpeg.exec(["-i", inputName, "-f", "null", "-"]);
} catch (error) {
listening = false;
await ffmpeg.deleteFile(inputName);
console.error("FFmpeg execution failed:", error);
throw new Error(
"Failed to extract video info. The file may be corrupted or in an unsupported format."
);
if (!videoTrack) {
throw new Error("No video track found in the file");
}
// Disable listener after exec completes
listening = false;
// Cleanup
await ffmpeg.deleteFile(inputName);
// Parse output for duration, resolution, and fps
// Example: Duration: 00:00:10.00, start: 0.000000, bitrate: 1234 kb/s
// Example: Stream #0:0: Video: h264 (High), yuv420p(progressive), 1920x1080 [SAR 1:1 DAR 16:9], 30 fps, 30 tbr, 90k tbn, 60 tbc
const durationMatch = ffmpegOutput.match(/Duration: (\d+):(\d+):([\d.]+)/);
let duration = 0;
if (durationMatch) {
const [, h, m, s] = durationMatch;
duration = parseInt(h) * 3600 + parseInt(m) * 60 + parseFloat(s);
}
const videoStreamMatch = ffmpegOutput.match(
/Video:.* (\d+)x(\d+)[^,]*, ([\d.]+) fps/
);
let width = 0,
height = 0,
fps = 0;
if (videoStreamMatch) {
width = parseInt(videoStreamMatch[1]);
height = parseInt(videoStreamMatch[2]);
fps = parseFloat(videoStreamMatch[3]);
}
// Get frame rate from packet statistics
const packetStats = await videoTrack.computePacketStats(100);
const fps = packetStats.averagePacketRate;
return {
duration,
width,
height,
width: videoTrack.displayWidth,
height: videoTrack.displayHeight,
fps,
};
};
export const convertToWebM = async (
videoFile: File,
onProgress?: (progress: number) => void
): Promise<Blob> => {
const ffmpeg = await initFFmpeg();
const inputName = "input.mp4";
const outputName = "output.webm";
// Set up progress callback
if (onProgress) {
ffmpeg.on("progress", ({ progress }) => {
onProgress(progress * 100);
});
}
// Write input file
await ffmpeg.writeFile(
inputName,
new Uint8Array(await videoFile.arrayBuffer())
);
// Convert to WebM
await ffmpeg.exec([
"-i",
inputName,
"-c:v",
"libvpx-vp9",
"-crf",
"30",
"-b:v",
"0",
"-c:a",
"libopus",
outputName,
]);
// Read output file
const data = await ffmpeg.readFile(outputName);
const blob = new Blob([data], { type: "video/webm" });
// Cleanup
await ffmpeg.deleteFile(inputName);
await ffmpeg.deleteFile(outputName);
return blob;
};
export const extractAudio = async (
videoFile: File,
format: "mp3" | "wav" = "mp3"
): Promise<Blob> => {
const ffmpeg = await initFFmpeg();
const inputName = "input.mp4";
const outputName = `output.${format}`;
// Write input file
await ffmpeg.writeFile(
inputName,
new Uint8Array(await videoFile.arrayBuffer())
);
// Extract audio
await ffmpeg.exec([
"-i",
inputName,
"-vn", // Disable video
"-acodec",
format === "mp3" ? "libmp3lame" : "pcm_s16le",
outputName,
]);
// Read output file
const data = await ffmpeg.readFile(outputName);
const blob = new Blob([data], { type: `audio/${format}` });
// Cleanup
await ffmpeg.deleteFile(inputName);
await ffmpeg.deleteFile(outputName);
return blob;
};
}
// Audio mixing for timeline - keeping FFmpeg for now due to complexity
// TODO: Replace with Mediabunny audio processing when implementing canvas preview
export const extractTimelineAudio = async (
onProgress?: (progress: number) => void
): Promise<Blob> => {

View File

@ -0,0 +1,179 @@
import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
import type { TimelineTrack } from "@/types/timeline";
import type { MediaItem } from "@/stores/media-store";
export interface RenderContext {
ctx: CanvasRenderingContext2D;
time: number;
canvasWidth: number;
canvasHeight: number;
tracks: TimelineTrack[];
mediaItems: MediaItem[];
backgroundColor?: string;
useCache?: boolean;
cache?: Map<
string,
Map<
number,
{
draw: (
c: CanvasRenderingContext2D,
x: number,
y: number,
w: number,
h: number
) => void;
}
>
>;
fps: number;
}
export async function renderTimelineFrame({
ctx,
time,
canvasWidth,
canvasHeight,
tracks,
mediaItems,
backgroundColor,
useCache = true,
cache,
fps,
}: RenderContext): Promise<void> {
// Background
ctx.clearRect(0, 0, canvasWidth, canvasHeight);
if (backgroundColor && backgroundColor !== "transparent") {
ctx.fillStyle = backgroundColor;
ctx.fillRect(0, 0, canvasWidth, canvasHeight);
}
const scaleX = 1;
const scaleY = 1;
const idToMedia = new Map(mediaItems.map((m) => [m.id, m] as const));
const active: Array<{
track: TimelineTrack;
element: TimelineTrack["elements"][number];
mediaItem: MediaItem | null;
}> = [];
for (let t = tracks.length - 1; t >= 0; t -= 1) {
const track = tracks[t];
for (const element of track.elements) {
if ((element as any).hidden) continue;
const elementStart = element.startTime;
const elementEnd =
element.startTime +
(element.duration - element.trimStart - element.trimEnd);
if (time >= elementStart && time < elementEnd) {
let mediaItem: MediaItem | null = null;
if (element.type === "media") {
mediaItem =
element.mediaId === "test"
? null
: idToMedia.get(element.mediaId) || null;
}
active.push({ track, element, mediaItem });
}
}
}
for (const { element, mediaItem } of active) {
if (element.type === "media" && mediaItem) {
if (mediaItem.type === "video") {
const input = new Input({
source: new BlobSource(mediaItem.file),
formats: ALL_FORMATS,
});
const track = await input.getPrimaryVideoTrack();
if (!track) continue;
const decodable = await track.canDecode();
if (!decodable) continue;
const sink = new VideoSampleSink(track);
const localTime = time - element.startTime + element.trimStart;
const sample = await sink.getSample(localTime);
if (!sample) continue;
const mediaW = Math.max(1, mediaItem.width || canvasWidth);
const mediaH = Math.max(1, mediaItem.height || canvasHeight);
const containScale = Math.min(
canvasWidth / mediaW,
canvasHeight / mediaH
);
const drawW = mediaW * containScale;
const drawH = mediaH * containScale;
const drawX = (canvasWidth - drawW) / 2;
const drawY = (canvasHeight - drawH) / 2;
sample.draw(ctx, drawX, drawY, drawW, drawH);
}
if (mediaItem.type === "image") {
const img = new Image();
await new Promise<void>((resolve, reject) => {
img.onload = () => resolve();
img.onerror = () => reject(new Error("Image load failed"));
img.src = mediaItem.url || URL.createObjectURL(mediaItem.file);
});
const mediaW = Math.max(
1,
mediaItem.width || img.naturalWidth || canvasWidth
);
const mediaH = Math.max(
1,
mediaItem.height || img.naturalHeight || canvasHeight
);
const containScale = Math.min(
canvasWidth / mediaW,
canvasHeight / mediaH
);
const drawW = mediaW * containScale;
const drawH = mediaH * containScale;
const drawX = (canvasWidth - drawW) / 2;
const drawY = (canvasHeight - drawH) / 2;
ctx.drawImage(img, drawX, drawY, drawW, drawH);
}
}
if (element.type === "text") {
const posX = canvasWidth / 2 + (element as any).x * scaleX;
const posY = canvasHeight / 2 + (element as any).y * scaleY;
ctx.save();
ctx.translate(posX, posY);
ctx.rotate(((element as any).rotation * Math.PI) / 180);
ctx.globalAlpha = Math.max(0, Math.min(1, (element as any).opacity));
const px = (element as any).fontSize * scaleX;
const weight = (element as any).fontWeight === "bold" ? "bold " : "";
const style = (element as any).fontStyle === "italic" ? "italic " : "";
ctx.font = `${style}${weight}${px}px ${(element as any).fontFamily}`;
ctx.fillStyle = (element as any).color;
ctx.textAlign = (element as any).textAlign;
ctx.textBaseline = "middle";
const metrics = ctx.measureText((element as any).content);
const ascent =
(metrics as unknown as { actualBoundingBoxAscent?: number })
.actualBoundingBoxAscent ?? px * 0.8;
const descent =
(metrics as unknown as { actualBoundingBoxDescent?: number })
.actualBoundingBoxDescent ?? px * 0.2;
const textW = metrics.width;
const textH = ascent + descent;
const padX = 8 * scaleX;
const padY = 4 * scaleX;
if ((element as any).backgroundColor) {
ctx.save();
ctx.fillStyle = (element as any).backgroundColor;
let bgLeft = -textW / 2;
if (ctx.textAlign === "left") bgLeft = 0;
if (ctx.textAlign === "right") bgLeft = -textW;
ctx.fillRect(
bgLeft - padX,
-textH / 2 - padY,
textW + padX * 2,
textH + padY * 2
);
ctx.restore();
}
ctx.fillText((element as any).content, 0, 0);
ctx.restore();
}
}
}

View File

@ -4,6 +4,7 @@
"": {
"name": "opencut",
"dependencies": {
"mediabunny": "^1.9.3",
"next": "^15.3.4",
"react-country-flag": "^3.1.0",
"wavesurfer.js": "^7.9.8",
@ -558,6 +559,10 @@
"@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="],
"@types/dom-mediacapture-transform": ["@types/dom-mediacapture-transform@0.1.11", "", { "dependencies": { "@types/dom-webcodecs": "*" } }, "sha512-Y2p+nGf1bF2XMttBnsVPHUWzRRZzqUoJAKmiP10b5umnO6DDrWI0BrGDJy1pOHoOULVmGSfFNkQrAlC5dcj6nQ=="],
"@types/dom-webcodecs": ["@types/dom-webcodecs@0.1.13", "", {}, "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ=="],
"@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="],
"@types/estree-jsx": ["@types/estree-jsx@1.0.5", "", { "dependencies": { "@types/estree": "*" } }, "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg=="],
@ -896,6 +901,8 @@
"mdast-util-to-string": ["mdast-util-to-string@4.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0" } }, "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg=="],
"mediabunny": ["mediabunny@1.9.3", "", { "dependencies": { "@types/dom-mediacapture-transform": "^0.1.11", "@types/dom-webcodecs": "0.1.13" } }, "sha512-uVspdzrpwvW8NxtGspHPOJvdNSY0dNFEmOQngSheX9PYX9oXijZENBjfYWOgm9c9IgSSlN2bdJF9iyPyY3f+fQ=="],
"micromark": ["micromark@4.0.2", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA=="],
"micromark-core-commonmark": ["micromark-core-commonmark@2.0.3", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-destination": "^2.0.0", "micromark-factory-label": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-factory-title": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg=="],
@ -1320,10 +1327,10 @@
"motion/framer-motion/motion-utils": ["motion-utils@12.23.6", "", {}, "sha512-eAWoPgr4eFEOFfg2WjIsMoqJTW6Z8MTUCgn/GZ3VRpClWBdnbjryiA3ZSNLyxCTmCQx4RmYX6jX1iWHbenUPNQ=="],
"opencut/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="],
"next/postcss/nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],
"opencut/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="],
"opencut/next/@next/env": ["@next/env@15.4.5", "", {}, "sha512-ruM+q2SCOVCepUiERoxOmZY9ZVoecR3gcXNwCYZRvQQWRjhOiPJGmQ2fAiLR6YKWXcSAh7G79KEFxN3rwhs4LQ=="],
"opencut/next/@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@15.4.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-84dAN4fkfdC7nX6udDLz9GzQlMUwEMKD7zsseXrl7FTeIItF8vpk1lhLEnsotiiDt+QFu3O1FVWnqwcRD2U3KA=="],

View File

@ -20,6 +20,7 @@
"format": "npx ultracite@latest format"
},
"dependencies": {
"mediabunny": "^1.9.3",
"next": "^15.3.4",
"react-country-flag": "^3.1.0",
"wavesurfer.js": "^7.9.8"