diff --git a/apps/web/src/services/media/video-cache.ts b/apps/web/src/services/media/video-cache.ts index 48c8f56c..f3ddfcb2 100644 --- a/apps/web/src/services/media/video-cache.ts +++ b/apps/web/src/services/media/video-cache.ts @@ -10,7 +10,10 @@ interface VideoSinkData { sink: CanvasSink; iterator: AsyncGenerator | null; currentFrame: WrappedCanvas | null; + nextFrame: WrappedCanvas | null; lastTime: number; + prefetching: boolean; + prefetchPromise: Promise | null; } export class VideoCache { private sinks = new Map(); @@ -30,10 +33,19 @@ export class VideoCache { const sinkData = this.sinks.get(mediaId); if (!sinkData) return null; + if (sinkData.nextFrame && sinkData.nextFrame.timestamp <= time) { + sinkData.currentFrame = sinkData.nextFrame; + sinkData.nextFrame = null; + this.startPrefetch({ sinkData }); + } + if ( sinkData.currentFrame && this.isFrameValid({ frame: sinkData.currentFrame, time }) ) { + if (!sinkData.nextFrame && !sinkData.prefetching) { + this.startPrefetch({ sinkData }); + } return sinkData.currentFrame; } @@ -44,10 +56,19 @@ export class VideoCache { time < sinkData.lastTime + 2.0 ) { const frame = await this.iterateToTime({ sinkData, targetTime: time }); - if (frame) return frame; + if (frame) { + if (!sinkData.nextFrame && !sinkData.prefetching) { + this.startPrefetch({ sinkData }); + } + return frame; + } } - return await this.seekToTime({ sinkData, time }); + const frame = await this.seekToTime({ sinkData, time }); + if (frame && !sinkData.nextFrame && !sinkData.prefetching) { + this.startPrefetch({ sinkData }); + } + return frame; } private isFrameValid({ @@ -70,11 +91,29 @@ export class VideoCache { try { while (true) { - const { value: frame, done } = await sinkData.iterator.next(); + // Wait for any pending prefetch to finish before touching iterator + if (sinkData.prefetching && sinkData.prefetchPromise) { + await sinkData.prefetchPromise; + } - if (done || !frame) break; + // Check if the nextFrame (which might have just arrived) is what we need + if ( + sinkData.nextFrame && + sinkData.nextFrame.timestamp <= targetTime + 0.05 // Tolerance + ) { + sinkData.currentFrame = sinkData.nextFrame; + sinkData.nextFrame = null; + } else { + const { value: frame, done } = await sinkData.iterator.next(); + + if (done || !frame) break; + + sinkData.currentFrame = frame; + } + + const frame = sinkData.currentFrame; + if (!frame) break; - sinkData.currentFrame = frame; sinkData.lastTime = frame.timestamp; if (this.isFrameValid({ frame, time: targetTime })) { @@ -98,18 +137,36 @@ export class VideoCache { time: number; }): Promise { try { + if (sinkData.prefetching && sinkData.prefetchPromise) { + await sinkData.prefetchPromise; + } + if (sinkData.iterator) { await sinkData.iterator.return(); sinkData.iterator = null; } + sinkData.nextFrame = null; sinkData.iterator = sinkData.sink.canvases(time); sinkData.lastTime = time; + // Fetch current frame const { value: frame } = await sinkData.iterator.next(); if (frame) { sinkData.currentFrame = frame; + + // Aggressively fetch next frame immediately to fill buffer + // This matches the mediaplayer example which fetches 2 frames on start + try { + const { value: next } = await sinkData.iterator.next(); + if (next) { + sinkData.nextFrame = next; + } + } catch (e) { + console.warn("Failed to pre-fetch next frame on seek:", e); + } + return frame; } } catch (error) { @@ -118,6 +175,46 @@ export class VideoCache { return null; } + + private startPrefetch({ sinkData }: { sinkData: VideoSinkData }): void { + if (sinkData.prefetching || !sinkData.iterator || sinkData.nextFrame) { + return; + } + + sinkData.prefetching = true; + sinkData.prefetchPromise = this.prefetchNextFrame({ sinkData }); + } + + private async prefetchNextFrame({ + sinkData, + }: { + sinkData: VideoSinkData; + }): Promise { + if (!sinkData.iterator) { + sinkData.prefetching = false; + sinkData.prefetchPromise = null; + return; + } + + try { + const { value: frame, done } = await sinkData.iterator.next(); + + if (done || !frame) { + sinkData.prefetching = false; + sinkData.prefetchPromise = null; + return; + } + + sinkData.nextFrame = frame; + sinkData.prefetching = false; + sinkData.prefetchPromise = null; + } catch (error) { + console.warn("Prefetch failed:", error); + sinkData.prefetching = false; + sinkData.prefetchPromise = null; + sinkData.iterator = null; + } + } private async ensureSink({ mediaId, file, @@ -173,7 +270,10 @@ export class VideoCache { sink, iterator: null, currentFrame: null, + nextFrame: null, lastTime: -1, + prefetching: false, + prefetchPromise: null, }); } catch (error) { console.error(`Failed to initialize video sink for ${mediaId}:`, error); @@ -185,7 +285,7 @@ export class VideoCache { const sinkData = this.sinks.get(mediaId); if (sinkData) { if (sinkData.iterator) { - sinkData.iterator.return(); + void sinkData.iterator.return(); } this.sinks.delete(mediaId); diff --git a/apps/web/src/services/renderer/nodes/video-node.ts b/apps/web/src/services/renderer/nodes/video-node.ts index e22c9082..ac275dc2 100644 --- a/apps/web/src/services/renderer/nodes/video-node.ts +++ b/apps/web/src/services/renderer/nodes/video-node.ts @@ -1,6 +1,6 @@ -import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny"; import type { CanvasRenderer } from "../canvas-renderer"; import { BaseNode } from "./base-node"; +import { videoCache } from "@/services/media/video-cache"; const VIDEO_EPSILON = 1 / 1000; @@ -17,33 +17,13 @@ export interface BaseMediaNodeParams { opacity?: number; } -export type VideoNodeParams = BaseMediaNodeParams; +export interface VideoNodeParams extends BaseMediaNodeParams { + mediaId: string; +} export class VideoNode extends BaseNode { - private sink?: VideoSampleSink; - private readyPromise: Promise; - constructor(params: VideoNodeParams) { super(params); - this.readyPromise = this.load(); - } - - private async load() { - const input = new Input({ - source: new BlobSource(this.params.file), - formats: ALL_FORMATS, - }); - - const videoTrack = await input.getPrimaryVideoTrack(); - if (!videoTrack) { - throw new Error("No video track found"); - } - - if (!(await videoTrack.canDecode())) { - throw new Error("Unable to decode the video track."); - } - - this.sink = new VideoSampleSink(videoTrack); } private getVideoTime(time: number) { @@ -65,44 +45,38 @@ export class VideoNode extends BaseNode { return; } - await this.readyPromise; - - if (!this.sink) { - throw new Error("Sink not initialized"); - } - const videoTime = this.getVideoTime(time); - const sample = await this.sink.getSample(videoTime); + const frame = await videoCache.getFrameAt({ + mediaId: this.params.mediaId, + file: this.params.file, + time: videoTime, + }); - if (sample) { - try { - renderer.context.save(); + if (frame) { + renderer.context.save(); - if (this.params.opacity !== undefined) { - renderer.context.globalAlpha = this.params.opacity; - } - - if ( - this.params.x !== undefined && - this.params.y !== undefined && - this.params.width !== undefined && - this.params.height !== undefined - ) { - sample.draw( - renderer.context, - this.params.x, - this.params.y, - this.params.width, - this.params.height, - ); - } else { - sample.draw(renderer.context, 0, 0, renderer.width, renderer.height); - } - - renderer.context.restore(); - } finally { - sample.close(); + if (this.params.opacity !== undefined) { + renderer.context.globalAlpha = this.params.opacity; } + + if ( + this.params.x !== undefined && + this.params.y !== undefined && + this.params.width !== undefined && + this.params.height !== undefined + ) { + renderer.context.drawImage( + frame.canvas, + this.params.x, + this.params.y, + this.params.width, + this.params.height, + ); + } else { + renderer.context.drawImage(frame.canvas, 0, 0, renderer.width, renderer.height); + } + + renderer.context.restore(); } } } diff --git a/apps/web/src/services/renderer/scene-builder.ts b/apps/web/src/services/renderer/scene-builder.ts index 01827945..b36edda1 100644 --- a/apps/web/src/services/renderer/scene-builder.ts +++ b/apps/web/src/services/renderer/scene-builder.ts @@ -58,6 +58,7 @@ export function buildScene(params: BuildSceneParams) { if (mediaAsset.type === "video") { contentNodes.push( new VideoNode({ + mediaId: mediaAsset.id, file: mediaAsset.file, duration: element.duration, timeOffset: element.startTime,