From d747fb458d2fbebd2ce007132b43cd973bd5513f Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Sun, 31 Aug 2025 13:40:41 +0200 Subject: [PATCH] feat: make timeline renderer 15x faster --- apps/web/src/lib/timeline-renderer.ts | 51 +++---- apps/web/src/lib/video-cache.ts | 186 ++++++++++++++++++++++++++ apps/web/src/stores/media-store.ts | 5 + 3 files changed, 218 insertions(+), 24 deletions(-) create mode 100644 apps/web/src/lib/video-cache.ts diff --git a/apps/web/src/lib/timeline-renderer.ts b/apps/web/src/lib/timeline-renderer.ts index 6241a7d9..08785369 100644 --- a/apps/web/src/lib/timeline-renderer.ts +++ b/apps/web/src/lib/timeline-renderer.ts @@ -1,6 +1,6 @@ -import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny"; import type { TimelineTrack } from "@/types/timeline"; import type { MediaFile } from "@/types/media"; +import { videoCache } from "./video-cache"; export interface RenderContext { ctx: CanvasRenderingContext2D; @@ -65,31 +65,34 @@ export async function renderTimelineFrame({ for (const { element, mediaItem } of active) { if (element.type === "media" && mediaItem) { if (mediaItem.type === "video") { - const input = new Input({ - source: new BlobSource(mediaItem.file), - formats: ALL_FORMATS, - }); - const track = await input.getPrimaryVideoTrack(); - if (!track) continue; - const decodable = await track.canDecode(); - if (!decodable) continue; - const sink = new VideoSampleSink(track); + try { + const localTime = time - element.startTime + element.trimStart; - const localTime = time - element.startTime + element.trimStart; - const sample = await sink.getSample(localTime); - if (!sample) continue; + const frame = await videoCache.getFrameAt( + mediaItem.id, + mediaItem.file, + localTime + ); + if (!frame) continue; - const mediaW = Math.max(1, mediaItem.width || canvasWidth); - const mediaH = Math.max(1, mediaItem.height || canvasHeight); - const containScale = Math.min( - canvasWidth / mediaW, - canvasHeight / mediaH - ); - const drawW = mediaW * containScale; - const drawH = mediaH * containScale; - const drawX = (canvasWidth - drawW) / 2; - const drawY = (canvasHeight - drawH) / 2; - sample.draw(ctx, drawX, drawY, drawW, drawH); + const mediaW = Math.max(1, mediaItem.width || canvasWidth); + const mediaH = Math.max(1, mediaItem.height || canvasHeight); + const containScale = Math.min( + canvasWidth / mediaW, + canvasHeight / mediaH + ); + const drawW = mediaW * containScale; + const drawH = mediaH * containScale; + const drawX = (canvasWidth - drawW) / 2; + const drawY = (canvasHeight - drawH) / 2; + + ctx.drawImage(frame.canvas, drawX, drawY, drawW, drawH); + } catch (error) { + console.warn( + `Failed to render video frame for ${mediaItem.name}:`, + error + ); + } } if (mediaItem.type === "image") { const img = new Image(); diff --git a/apps/web/src/lib/video-cache.ts b/apps/web/src/lib/video-cache.ts new file mode 100644 index 00000000..aeb62d1c --- /dev/null +++ b/apps/web/src/lib/video-cache.ts @@ -0,0 +1,186 @@ +import { + Input, + ALL_FORMATS, + BlobSource, + CanvasSink, + WrappedCanvas, +} from "mediabunny"; + +interface VideoSinkData { + sink: CanvasSink; + iterator: AsyncGenerator | null; + currentFrame: WrappedCanvas | null; + lastTime: number; +} +export class VideoCache { + private sinks = new Map(); + private initPromises = new Map>(); + + async getFrameAt( + mediaId: string, + file: File, + time: number + ): Promise { + await this.ensureSink(mediaId, file); + + const sinkData = this.sinks.get(mediaId); + if (!sinkData) return null; + + if ( + sinkData.currentFrame && + this.isFrameValid(sinkData.currentFrame, time) + ) { + return sinkData.currentFrame; + } + + if ( + sinkData.iterator && + sinkData.currentFrame && + time >= sinkData.lastTime && + time < sinkData.lastTime + 2.0 + ) { + const frame = await this.iterateToTime(sinkData, time); + if (frame) return frame; + } + + return await this.seekToTime(sinkData, time); + } + + private isFrameValid(frame: WrappedCanvas, time: number): boolean { + return time >= frame.timestamp && time < frame.timestamp + frame.duration; + } + private async iterateToTime( + sinkData: VideoSinkData, + targetTime: number + ): Promise { + if (!sinkData.iterator) return null; + + try { + while (true) { + const { value: frame, done } = await sinkData.iterator.next(); + + if (done || !frame) break; + + sinkData.currentFrame = frame; + sinkData.lastTime = frame.timestamp; + + if (this.isFrameValid(frame, targetTime)) { + return frame; + } + + if (frame.timestamp > targetTime + 1.0) break; + } + } catch (error) { + console.warn("Iterator failed, will restart:", error); + sinkData.iterator = null; + } + + return null; + } + private async seekToTime( + sinkData: VideoSinkData, + time: number + ): Promise { + try { + if (sinkData.iterator) { + await sinkData.iterator.return(); + sinkData.iterator = null; + } + + sinkData.iterator = sinkData.sink.canvases(time); + sinkData.lastTime = time; + + const { value: frame } = await sinkData.iterator.next(); + + if (frame) { + sinkData.currentFrame = frame; + return frame; + } + } catch (error) { + console.warn("Failed to seek video:", error); + } + + return null; + } + private async ensureSink(mediaId: string, file: File): Promise { + if (this.sinks.has(mediaId)) return; + + if (this.initPromises.has(mediaId)) { + await this.initPromises.get(mediaId); + return; + } + + const initPromise = this.initializeSink(mediaId, file); + this.initPromises.set(mediaId, initPromise); + + try { + await initPromise; + } finally { + this.initPromises.delete(mediaId); + } + } + private async initializeSink(mediaId: string, file: File): Promise { + try { + const input = new Input({ + source: new BlobSource(file), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + if (!videoTrack) { + throw new Error("No video track found"); + } + + const canDecode = await videoTrack.canDecode(); + if (!canDecode) { + throw new Error("Video codec not supported for decoding"); + } + + const sink = new CanvasSink(videoTrack, { + poolSize: 3, + fit: "contain", + }); + + this.sinks.set(mediaId, { + sink, + iterator: null, + currentFrame: null, + lastTime: -1, + }); + } catch (error) { + console.error(`Failed to initialize video sink for ${mediaId}:`, error); + throw error; + } + } + + clearVideo(mediaId: string): void { + const sinkData = this.sinks.get(mediaId); + if (sinkData) { + if (sinkData.iterator) { + sinkData.iterator.return(); + } + + this.sinks.delete(mediaId); + } + + this.initPromises.delete(mediaId); + } + + clearAll(): void { + for (const [mediaId] of this.sinks) { + this.clearVideo(mediaId); + } + } + + getStats() { + return { + totalSinks: this.sinks.size, + activeSinks: Array.from(this.sinks.values()).filter((s) => s.iterator) + .length, + cachedFrames: Array.from(this.sinks.values()).filter( + (s) => s.currentFrame + ).length, + }; + } +} +export const videoCache = new VideoCache(); diff --git a/apps/web/src/stores/media-store.ts b/apps/web/src/stores/media-store.ts index 2b7cb62e..c9e29fff 100644 --- a/apps/web/src/stores/media-store.ts +++ b/apps/web/src/stores/media-store.ts @@ -3,6 +3,7 @@ import { storageService } from "@/lib/storage/storage-service"; import { useTimelineStore } from "./timeline-store"; import { generateUUID } from "@/lib/utils"; import { MediaType, MediaFile } from "@/types/media"; +import { videoCache } from "@/lib/video-cache"; interface MediaStore { mediaFiles: MediaFile[]; @@ -165,6 +166,8 @@ export const useMediaStore = create((set, get) => ({ const state = get(); const item = state.mediaFiles.find((media) => media.id === id); + videoCache.clearVideo(id); + // Cleanup object URLs to prevent memory leaks if (item?.url) { URL.revokeObjectURL(item.url); @@ -289,6 +292,8 @@ export const useMediaStore = create((set, get) => ({ clearAllMedia: () => { const state = get(); + videoCache.clearAll(); + // Cleanup all object URLs state.mediaFiles.forEach((item) => { if (item.url) {