fix video node lag

apps/web/src/services/media/video-cache.ts

@@ -10,7 +10,10 @@ interface VideoSinkData {
 	sink: CanvasSink;
 	iterator: AsyncGenerator<WrappedCanvas, void, unknown> | null;
 	currentFrame: WrappedCanvas | null;
+	nextFrame: WrappedCanvas | null;
 	lastTime: number;
+	prefetching: boolean;
+	prefetchPromise: Promise<void> | null;
 }
 export class VideoCache {
 	private sinks = new Map<string, VideoSinkData>();
@@ -30,10 +33,19 @@ export class VideoCache {
 		const sinkData = this.sinks.get(mediaId);
 		if (!sinkData) return null;
 
+		if (sinkData.nextFrame && sinkData.nextFrame.timestamp <= time) {
+			sinkData.currentFrame = sinkData.nextFrame;
+			sinkData.nextFrame = null;
+			this.startPrefetch({ sinkData });
+		}
+
 		if (
 			sinkData.currentFrame &&
 			this.isFrameValid({ frame: sinkData.currentFrame, time })
 		) {
+			if (!sinkData.nextFrame && !sinkData.prefetching) {
+				this.startPrefetch({ sinkData });
+			}
 			return sinkData.currentFrame;
 		}
 
@@ -44,10 +56,19 @@ export class VideoCache {
 			time < sinkData.lastTime + 2.0
 		) {
 			const frame = await this.iterateToTime({ sinkData, targetTime: time });
-			if (frame) return frame;
+			if (frame) {
+				if (!sinkData.nextFrame && !sinkData.prefetching) {
+					this.startPrefetch({ sinkData });
+				}
+				return frame;
+			}
 		}
 
-		return await this.seekToTime({ sinkData, time });
+		const frame = await this.seekToTime({ sinkData, time });
+		if (frame && !sinkData.nextFrame && !sinkData.prefetching) {
+			this.startPrefetch({ sinkData });
+		}
+		return frame;
 	}
 
 	private isFrameValid({
@@ -70,11 +91,29 @@ export class VideoCache {
 
 		try {
 			while (true) {
-				const { value: frame, done } = await sinkData.iterator.next();
+				// Wait for any pending prefetch to finish before touching iterator
+				if (sinkData.prefetching && sinkData.prefetchPromise) {
+					await sinkData.prefetchPromise;
+				}
 
-				if (done || !frame) break;
+				// Check if the nextFrame (which might have just arrived) is what we need
+				if (
+					sinkData.nextFrame &&
+					sinkData.nextFrame.timestamp <= targetTime + 0.05 // Tolerance
+				) {
+					sinkData.currentFrame = sinkData.nextFrame;
+					sinkData.nextFrame = null;
+				} else {
+					const { value: frame, done } = await sinkData.iterator.next();
+
+					if (done || !frame) break;
+
+					sinkData.currentFrame = frame;
+				}
+
+				const frame = sinkData.currentFrame;
+				if (!frame) break;
 
-				sinkData.currentFrame = frame;
 				sinkData.lastTime = frame.timestamp;
 
 				if (this.isFrameValid({ frame, time: targetTime })) {
@@ -98,18 +137,36 @@ export class VideoCache {
 		time: number;
 	}): Promise<WrappedCanvas | null> {
 		try {
+			if (sinkData.prefetching && sinkData.prefetchPromise) {
+				await sinkData.prefetchPromise;
+			}
+
 			if (sinkData.iterator) {
 				await sinkData.iterator.return();
 				sinkData.iterator = null;
 			}
 
+			sinkData.nextFrame = null;
 			sinkData.iterator = sinkData.sink.canvases(time);
 			sinkData.lastTime = time;
 
+			// Fetch current frame
 			const { value: frame } = await sinkData.iterator.next();
 
 			if (frame) {
 				sinkData.currentFrame = frame;
+
+				// Aggressively fetch next frame immediately to fill buffer
+				// This matches the mediaplayer example which fetches 2 frames on start
+				try {
+					const { value: next } = await sinkData.iterator.next();
+					if (next) {
+						sinkData.nextFrame = next;
+					}
+				} catch (e) {
+					console.warn("Failed to pre-fetch next frame on seek:", e);
+				}
+
 				return frame;
 			}
 		} catch (error) {
@@ -118,6 +175,46 @@ export class VideoCache {
 
 		return null;
 	}
+
+	private startPrefetch({ sinkData }: { sinkData: VideoSinkData }): void {
+		if (sinkData.prefetching || !sinkData.iterator || sinkData.nextFrame) {
+			return;
+		}
+
+		sinkData.prefetching = true;
+		sinkData.prefetchPromise = this.prefetchNextFrame({ sinkData });
+	}
+
+	private async prefetchNextFrame({
+		sinkData,
+	}: {
+		sinkData: VideoSinkData;
+	}): Promise<void> {
+		if (!sinkData.iterator) {
+			sinkData.prefetching = false;
+			sinkData.prefetchPromise = null;
+			return;
+		}
+
+		try {
+			const { value: frame, done } = await sinkData.iterator.next();
+
+			if (done || !frame) {
+				sinkData.prefetching = false;
+				sinkData.prefetchPromise = null;
+				return;
+			}
+
+			sinkData.nextFrame = frame;
+			sinkData.prefetching = false;
+			sinkData.prefetchPromise = null;
+		} catch (error) {
+			console.warn("Prefetch failed:", error);
+			sinkData.prefetching = false;
+			sinkData.prefetchPromise = null;
+			sinkData.iterator = null;
+		}
+	}
 	private async ensureSink({
 		mediaId,
 		file,
@@ -173,7 +270,10 @@ export class VideoCache {
 				sink,
 				iterator: null,
 				currentFrame: null,
+				nextFrame: null,
 				lastTime: -1,
+				prefetching: false,
+				prefetchPromise: null,
 			});
 		} catch (error) {
 			console.error(`Failed to initialize video sink for ${mediaId}:`, error);
@@ -185,7 +285,7 @@ export class VideoCache {
 		const sinkData = this.sinks.get(mediaId);
 		if (sinkData) {
 			if (sinkData.iterator) {
-				sinkData.iterator.return();
+				void sinkData.iterator.return();
 			}
 
 			this.sinks.delete(mediaId);

apps/web/src/services/renderer/nodes/video-node.ts

@@ -1,6 +1,6 @@
-import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
 import type { CanvasRenderer } from "../canvas-renderer";
 import { BaseNode } from "./base-node";
+import { videoCache } from "@/services/media/video-cache";
 
 const VIDEO_EPSILON = 1 / 1000;
 
@@ -17,33 +17,13 @@ export interface BaseMediaNodeParams {
 	opacity?: number;
 }
 
-export type VideoNodeParams = BaseMediaNodeParams;
+export interface VideoNodeParams extends BaseMediaNodeParams {
+	mediaId: string;
+}
 
 export class VideoNode extends BaseNode<VideoNodeParams> {
-	private sink?: VideoSampleSink;
-	private readyPromise: Promise<void>;
-
 	constructor(params: VideoNodeParams) {
 		super(params);
-		this.readyPromise = this.load();
-	}
-
-	private async load() {
-		const input = new Input({
-			source: new BlobSource(this.params.file),
-			formats: ALL_FORMATS,
-		});
-
-		const videoTrack = await input.getPrimaryVideoTrack();
-		if (!videoTrack) {
-			throw new Error("No video track found");
-		}
-
-		if (!(await videoTrack.canDecode())) {
-			throw new Error("Unable to decode the video track.");
-		}
-
-		this.sink = new VideoSampleSink(videoTrack);
 	}
 
 	private getVideoTime(time: number) {
@@ -65,44 +45,38 @@ export class VideoNode extends BaseNode<VideoNodeParams> {
 			return;
 		}
 
-		await this.readyPromise;
-
-		if (!this.sink) {
-			throw new Error("Sink not initialized");
-		}
-
 		const videoTime = this.getVideoTime(time);
-		const sample = await this.sink.getSample(videoTime);
+		const frame = await videoCache.getFrameAt({
+			mediaId: this.params.mediaId,
+			file: this.params.file,
+			time: videoTime,
+		});
 
-		if (sample) {
-			try {
-				renderer.context.save();
+		if (frame) {
+			renderer.context.save();
 
-				if (this.params.opacity !== undefined) {
-					renderer.context.globalAlpha = this.params.opacity;
-				}
-
-				if (
-					this.params.x !== undefined &&
-					this.params.y !== undefined &&
-					this.params.width !== undefined &&
-					this.params.height !== undefined
-				) {
-					sample.draw(
-						renderer.context,
-						this.params.x,
-						this.params.y,
-						this.params.width,
-						this.params.height,
-					);
-				} else {
-					sample.draw(renderer.context, 0, 0, renderer.width, renderer.height);
-				}
-
-				renderer.context.restore();
-			} finally {
-				sample.close();
+			if (this.params.opacity !== undefined) {
+				renderer.context.globalAlpha = this.params.opacity;
 			}
+
+			if (
+				this.params.x !== undefined &&
+				this.params.y !== undefined &&
+				this.params.width !== undefined &&
+				this.params.height !== undefined
+			) {
+				renderer.context.drawImage(
+					frame.canvas,
+					this.params.x,
+					this.params.y,
+					this.params.width,
+					this.params.height,
+				);
+			} else {
+				renderer.context.drawImage(frame.canvas, 0, 0, renderer.width, renderer.height);
+			}
+
+			renderer.context.restore();
 		}
 	}
 }

apps/web/src/services/renderer/scene-builder.ts

@@ -58,6 +58,7 @@ export function buildScene(params: BuildSceneParams) {
 				if (mediaAsset.type === "video") {
 					contentNodes.push(
 						new VideoNode({
+							mediaId: mediaAsset.id,
 							file: mediaAsset.file,
 							duration: element.duration,
 							timeOffset: element.startTime,