import { ALL_FORMATS, AudioBufferSink, BlobSource, CanvasSink, Input, UrlSource, type WrappedAudioBuffer, type WrappedCanvas, } from "mediabunny"; import SampleFileUrl from "../../docs/assets/big-buck-bunny-trimmed.mp4"; (document.querySelector("#sample-file-download") as HTMLAnchorElement).href = SampleFileUrl; const selectMediaButton = document.querySelector( "#select-file", ) as HTMLButtonElement; const loadUrlButton = document.querySelector("#load-url") as HTMLButtonElement; const fileNameElement = document.querySelector( "#file-name", ) as HTMLParagraphElement; const horizontalRule = document.querySelector("hr") as HTMLHRElement; const loadingElement = document.querySelector( "#loading-element", ) as HTMLParagraphElement; const playerContainer = document.querySelector("#player") as HTMLDivElement; const canvas = document.querySelector("canvas") as HTMLCanvasElement; const controlsElement = document.querySelector("#controls") as HTMLDivElement; const playButton = document.querySelector("#play-button") as HTMLButtonElement; const playIcon = document.querySelector("#play-icon") as HTMLSpanElement; const pauseIcon = document.querySelector("#pause-icon") as HTMLSpanElement; const currentTimeElement = document.querySelector( "#current-time", ) as HTMLSpanElement; const durationElement = document.querySelector("#duration") as HTMLSpanElement; const progressBarContainer = document.querySelector( "#progress-bar-container", ) as HTMLDivElement; const progressBar = document.querySelector("#progress-bar") as HTMLDivElement; const volumeBarContainer = document.querySelector( "#volume-bar-container", ) as HTMLDivElement; const volumeBar = document.querySelector("#volume-bar") as HTMLDivElement; const volumeIconWrapper = document.querySelector( "#volume-icon-wrapper", ) as HTMLDivElement; const volumeButton = document.querySelector( "#volume-button", ) as HTMLButtonElement; const fullscreenButton = document.querySelector( "#fullscreen-button", ) as HTMLButtonElement; const errorElement = document.querySelector("#error-element") as HTMLDivElement; const warningElement = document.querySelector( "#warning-element", ) as HTMLDivElement; const context = canvas.getContext("2d")!; let audioContext: AudioContext | null = null; let gainNode: GainNode | null = null; let fileLoaded = false; let videoSink: CanvasSink | null = null; let audioSink: AudioBufferSink | null = null; let totalDuration = 0; /** The value of the audio context's currentTime the moment the playback was started. */ let audioContextStartTime: number | null = null; let playing = false; /** The timestamp within the media file when the playback was started. */ let playbackTimeAtStart = 0; let videoFrameIterator: AsyncGenerator | null = null; let audioBufferIterator: AsyncGenerator< WrappedAudioBuffer, void, unknown > | null = null; let nextFrame: WrappedCanvas | null = null; const queuedAudioNodes: Set = new Set(); /** * Used to prevent async race conditions. When seekId is incremented, already-running async functions will be prevented * from having an effect. */ let asyncId = 0; let draggingProgressBar = false; let volume = 0.7; let draggingVolumeBar = false; let volumeMuted = false; /** === INIT LOGIC === */ const initMediaPlayer = async (resource: File | string) => { try { // First, dispose any ongoing playback: if (playing) { pause(); } void videoFrameIterator?.return(); void audioBufferIterator?.return(); asyncId++; fileLoaded = false; fileNameElement.textContent = resource instanceof File ? resource.name : resource; horizontalRule.style.display = ""; loadingElement.style.display = ""; playerContainer.style.display = "none"; errorElement.textContent = ""; warningElement.textContent = ""; // Create an Input from the resource const source = resource instanceof File ? new BlobSource(resource) : new UrlSource(resource); const input = new Input({ source, formats: ALL_FORMATS, }); playbackTimeAtStart = 0; totalDuration = await input.computeDuration(); durationElement.textContent = formatSeconds(totalDuration); let videoTrack = await input.getPrimaryVideoTrack(); let audioTrack = await input.getPrimaryAudioTrack(); let problemMessage = ""; if (videoTrack) { if (videoTrack.codec === null) { problemMessage += "Unsupported video codec. "; videoTrack = null; } else if (!(await videoTrack.canDecode())) { problemMessage += "Unable to decode the video track. "; videoTrack = null; } } if (audioTrack) { if (audioTrack.codec === null) { problemMessage += "Unsupported audio codec. "; audioTrack = null; } else if (!(await audioTrack.canDecode())) { problemMessage += "Unable to decode the audio track. "; audioTrack = null; } } if (!videoTrack && !audioTrack) { if (!problemMessage) { problemMessage = "No audio or video track found."; } throw new Error(problemMessage); } if (problemMessage) { warningElement.textContent = problemMessage; } // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access const AudioContext = window.AudioContext || (window as any).webkitAudioContext; // We must create the audio context with the matching sample rate for correct acoustic results // (especially for low-sample rate files) audioContext = new AudioContext({ sampleRate: audioTrack?.sampleRate }); gainNode = audioContext.createGain(); gainNode.connect(audioContext.destination); updateVolume(); const videoCanBeTransparent = videoTrack ? await videoTrack.canBeTransparent() : false; playerContainer.style.background = videoCanBeTransparent ? "transparent" : ""; // For video, let's use a CanvasSink as it handles rotation and closing video samples for us. // Pool size of 2: We'll only ever have the current and the next frame around, so we only need two canvases. videoSink = videoTrack && new CanvasSink(videoTrack, { poolSize: 2, fit: "contain", // In case the video changes dimensions over time alpha: videoCanBeTransparent, }); // For audio, we'll use an AudioBufferSink to directly retrieve AudioBuffers compatible with the Web Audio API audioSink = audioTrack && new AudioBufferSink(audioTrack); // Show the canvas if there's a video track, otherwise hide it if (videoTrack) { canvas.style.display = ""; canvas.width = videoTrack.displayWidth; canvas.height = videoTrack.displayHeight; } else { canvas.style.display = "none"; } // Show volume controls if there's an audio track, otherwise hide them if (audioTrack) { volumeButton.style.display = ""; volumeBarContainer.style.display = ""; } else { volumeButton.style.display = "none"; volumeBarContainer.style.display = "none"; } fileLoaded = true; await startVideoIterator(); if (audioContext.state === "running") { // Start playback automatically if the audio context permits await play(); } loadingElement.style.display = "none"; playerContainer.style.display = ""; if (!videoSink) { // If there's only an audio track, always show the controls controlsElement.style.opacity = "1"; controlsElement.style.pointerEvents = ""; playerContainer.style.cursor = ""; } } catch (error) { console.error(error); errorElement.textContent = String(error); loadingElement.style.display = "none"; playerContainer.style.display = "none"; } }; /** === VIDEO RENDERING LOGIC === */ /** Creates a new video frame iterator and renders the first video frame. */ const startVideoIterator = async () => { if (!videoSink) { return; } asyncId++; await videoFrameIterator?.return(); // Dispose of the current iterator // Create a new iterator videoFrameIterator = videoSink.canvases(getPlaybackTime()); // Get the first two frames const firstFrame = (await videoFrameIterator.next()).value ?? null; const secondFrame = (await videoFrameIterator.next()).value ?? null; nextFrame = secondFrame; if (firstFrame) { // Draw the first frame context.clearRect(0, 0, canvas.width, canvas.height); context.drawImage(firstFrame.canvas, 0, 0); } }; /** Runs every frame; updates the canvas if necessary. */ const render = (requestFrame = true) => { if (fileLoaded) { const playbackTime = getPlaybackTime(); if (playbackTime >= totalDuration) { // Pause playback once the end is reached pause(); playbackTimeAtStart = totalDuration; } // Check if the current playback time has caught up to the next frame if (nextFrame && nextFrame.timestamp <= playbackTime) { context.clearRect(0, 0, canvas.width, canvas.height); context.drawImage(nextFrame.canvas, 0, 0); nextFrame = null; // Request the next frame void updateNextFrame(); } if (!draggingProgressBar) { updateProgressBarTime(playbackTime); } } if (requestFrame) { requestAnimationFrame(() => render()); } }; render(); // Also call the render function on an interval to make sure the video keeps updating even if the tab isn't visible setInterval(() => render(false), 500); /** Iterates over the video frame iterator until it finds a video frame in the future. */ const updateNextFrame = async () => { const currentAsyncId = asyncId; // We have a loop here because we may need to iterate over multiple frames until we reach a frame in the future while (true) { const newNextFrame = (await videoFrameIterator!.next()).value ?? null; if (!newNextFrame) { break; } if (currentAsyncId !== asyncId) { break; } const playbackTime = getPlaybackTime(); if (newNextFrame.timestamp <= playbackTime) { // Draw it immediately context.clearRect(0, 0, canvas.width, canvas.height); context.drawImage(newNextFrame.canvas, 0, 0); } else { // Save it for later nextFrame = newNextFrame; break; } } }; /** === AUDIO PLAYBACK LOGIC === */ /** Loops over the audio buffer iterator, scheduling the audio to be played in the audio context. */ const runAudioIterator = async () => { if (!audioSink) { return; } // To play back audio, we loop over all audio chunks (typically very short) of the file and play them at the correct // timestamp. The result is a continuous, uninterrupted audio signal. for await (const { buffer, timestamp } of audioBufferIterator!) { const node = audioContext!.createBufferSource(); node.buffer = buffer; node.connect(gainNode!); const startTimestamp = audioContextStartTime! + timestamp - playbackTimeAtStart; // Two cases: Either, the audio starts in the future or in the past if (startTimestamp >= audioContext!.currentTime) { // If the audio starts in the future, easy, we just schedule it node.start(startTimestamp); } else { // If it starts in the past, then let's only play the audible section that remains from here on out node.start( audioContext!.currentTime, audioContext!.currentTime - startTimestamp, ); } queuedAudioNodes.add(node); node.onended = () => { queuedAudioNodes.delete(node); }; // If we're more than a second ahead of the current playback time, let's slow down the loop until time has // passed. if (timestamp - getPlaybackTime() >= 1) { await new Promise((resolve) => { const id = setInterval(() => { if (timestamp - getPlaybackTime() < 1) { clearInterval(id); resolve(); } }, 100); }); } } }; /** === PLAYBACK CONTROL LOGIC === */ /** Returns the current playback time in the media file. */ const getPlaybackTime = () => { if (playing) { // To ensure perfect audio-video sync, we always use the audio context's clock to determine playback time, even // when there is no audio track. return ( audioContext!.currentTime - audioContextStartTime! + playbackTimeAtStart ); } else { return playbackTimeAtStart; } }; const play = async () => { if (audioContext!.state === "suspended") { await audioContext!.resume(); } if (getPlaybackTime() === totalDuration) { // If we're at the end, let's snap back to the start playbackTimeAtStart = 0; await startVideoIterator(); } audioContextStartTime = audioContext!.currentTime; playing = true; if (audioSink) { // Start the audio iterator void audioBufferIterator?.return(); audioBufferIterator = audioSink?.buffers(getPlaybackTime()); void runAudioIterator(); } playIcon.style.display = "none"; pauseIcon.style.display = ""; }; const pause = () => { playbackTimeAtStart = getPlaybackTime(); playing = false; void audioBufferIterator?.return(); // This stops any for-loops that are iterating the iterator audioBufferIterator = null; // Stop all audio nodes that were already queued to play for (const node of queuedAudioNodes) { node.stop(); } queuedAudioNodes.clear(); playIcon.style.display = ""; pauseIcon.style.display = "none"; }; const togglePlay = () => { if (playing) { pause(); } else { void play(); } }; const seekToTime = async (seconds: number) => { updateProgressBarTime(seconds); const wasPlaying = playing; if (wasPlaying) { pause(); } playbackTimeAtStart = seconds; await startVideoIterator(); if (wasPlaying && playbackTimeAtStart < totalDuration) { void play(); } }; /** === PROGRESS BAR LOGIC === */ const updateProgressBarTime = (seconds: number) => { currentTimeElement.textContent = formatSeconds(seconds); progressBar.style.width = `${(seconds / totalDuration) * 100}%`; }; progressBarContainer.addEventListener("pointerdown", (event) => { draggingProgressBar = true; progressBarContainer.setPointerCapture(event.pointerId); const rect = progressBarContainer.getBoundingClientRect(); const completion = Math.max( Math.min((event.clientX - rect.left) / rect.width, 1), 0, ); updateProgressBarTime(completion * totalDuration); clearTimeout(hideControlsTimeout); window.addEventListener( "pointerup", (event) => { draggingProgressBar = false; progressBarContainer.releasePointerCapture(event.pointerId); const rect = progressBarContainer.getBoundingClientRect(); const completion = Math.max( Math.min((event.clientX - rect.left) / rect.width, 1), 0, ); const newTime = completion * totalDuration; void seekToTime(newTime); showControlsTemporarily(); }, { once: true }, ); }); progressBarContainer.addEventListener("pointermove", (event) => { if (draggingProgressBar) { const rect = progressBarContainer.getBoundingClientRect(); const completion = Math.max( Math.min((event.clientX - rect.left) / rect.width, 1), 0, ); updateProgressBarTime(completion * totalDuration); } }); /** === VOLUME CONTROL LOGIC === */ const updateVolume = () => { const actualVolume = volumeMuted ? 0 : volume; volumeBar.style.width = `${actualVolume * 100}%`; gainNode!.gain.value = actualVolume ** 2; // Quadratic for more fine-grained control const iconNumber = volumeMuted ? 0 : Math.ceil(1 + 3 * volume); for (let i = 0; i < volumeIconWrapper.children.length; i++) { const icon = volumeIconWrapper.children[i] as HTMLImageElement; icon.style.display = i === iconNumber ? "" : "none"; } }; volumeBarContainer.addEventListener("pointerdown", (event) => { draggingVolumeBar = true; volumeBarContainer.setPointerCapture(event.pointerId); const rect = volumeBarContainer.getBoundingClientRect(); volume = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0); volumeMuted = false; updateVolume(); clearTimeout(hideControlsTimeout); window.addEventListener( "pointerup", (event) => { draggingVolumeBar = false; volumeBarContainer.releasePointerCapture(event.pointerId); const rect = volumeBarContainer.getBoundingClientRect(); volume = Math.max( Math.min((event.clientX - rect.left) / rect.width, 1), 0, ); updateVolume(); showControlsTemporarily(); }, { once: true }, ); }); volumeButton.addEventListener("click", () => { volumeMuted = !volumeMuted; updateVolume(); }); volumeBarContainer.addEventListener("pointermove", (event) => { if (draggingVolumeBar) { const rect = volumeBarContainer.getBoundingClientRect(); volume = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0); updateVolume(); } }); /** === CONTROL UI LOGIC === */ const showControlsTemporarily = () => { if (!videoSink) { // Shouldn't run if there's only an audio track return; } controlsElement.style.opacity = "1"; controlsElement.style.pointerEvents = ""; playerContainer.style.cursor = ""; clearTimeout(hideControlsTimeout); hideControlsTimeout = window.setTimeout(() => { if (draggingProgressBar) { return; } hideControls(); playerContainer.style.cursor = "none"; }, 2000); }; const hideControls = () => { controlsElement.style.opacity = "0"; controlsElement.style.pointerEvents = "none"; }; hideControls(); let hideControlsTimeout = -1; playerContainer.addEventListener("pointermove", (event) => { if (event.pointerType !== "touch") { showControlsTemporarily(); } }); playerContainer.addEventListener("pointerleave", (event) => { if (!videoSink) { // Shouldn't run if there's only an audio track return; } if ( draggingProgressBar || draggingVolumeBar || event.pointerType === "touch" ) { return; } hideControls(); clearTimeout(hideControlsTimeout); }); /** === EVENT LISTENERS === */ playButton.addEventListener("click", togglePlay); window.addEventListener("keydown", (e) => { if (!fileLoaded) { return; } if (e.code === "Space" || e.code === "KeyK") { togglePlay(); } else if (e.code === "KeyF") { fullscreenButton.click(); } else if (e.code === "ArrowLeft") { const newTime = Math.max(getPlaybackTime() - 5, 0); void seekToTime(newTime); } else if (e.code === "ArrowRight") { const newTime = Math.min(getPlaybackTime() + 5, totalDuration); void seekToTime(newTime); } else if (e.code === "KeyM") { volumeButton.click(); } else { return; } showControlsTemporarily(); e.preventDefault(); }); fullscreenButton.addEventListener("click", () => { if (document.fullscreenElement) { void document.exitFullscreen(); } else { playerContainer.requestFullscreen().catch((e) => { console.error("Failed to enter fullscreen mode:", e); }); } }); // I'm sorry for this const isTouchDevice = () => { return "ontouchstart" in window; }; playerContainer.addEventListener("click", () => { if (isTouchDevice()) { if (controlsElement.style.opacity === "1") { hideControls(); } else { showControlsTemporarily(); } } else { togglePlay(); } }); controlsElement.addEventListener("click", (event) => { // Make sure this does NOT toggle play event.stopPropagation(); showControlsTemporarily(); }); /** === UTILS === */ const formatSeconds = (seconds: number) => { const showMilliseconds = window.innerWidth >= 640; seconds = Math.round(seconds * 1000) / 1000; // Round to milliseconds const hours = Math.floor(seconds / 3600); const minutes = Math.floor((seconds % 3600) / 60); const remainingSeconds = Math.floor(seconds % 60); const millisecs = Math.floor((1000 * seconds) % 1000) .toString() .padStart(3, "0"); let result: string; if (hours > 0) { result = `${hours}:${minutes.toString().padStart(2, "0")}` + `:${remainingSeconds.toString().padStart(2, "0")}`; } else { result = `${minutes.toString().padStart(2, "0")}:${remainingSeconds.toString().padStart(2, "0")}`; } if (showMilliseconds) { result += `.${millisecs}`; } return result; }; window.addEventListener("resize", () => { if (totalDuration) { updateProgressBarTime(getPlaybackTime()); durationElement.textContent = formatSeconds(totalDuration); } }); /** === FILE SELECTION LOGIC === */ selectMediaButton.addEventListener("click", () => { const fileInput = document.createElement("input"); fileInput.type = "file"; fileInput.accept = "video/*,video/x-matroska,video/mp2t,.ts,audio/*,audio/aac"; fileInput.addEventListener("change", () => { const file = fileInput.files?.[0]; if (!file) { return; } void initMediaPlayer(file); }); fileInput.click(); }); loadUrlButton.addEventListener("click", () => { const url = prompt( "Please enter a URL of a media file. Note that it must be HTTPS and support cross-origin requests, so have the" + " right CORS headers set.", "https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4", ); if (!url) { return; } void initMediaPlayer(url); }); document.addEventListener("dragover", (event) => { event.preventDefault(); event.dataTransfer!.dropEffect = "copy"; }); document.addEventListener("drop", (event) => { event.preventDefault(); const files = event.dataTransfer?.files; const file = files && files.length > 0 ? files[0] : undefined; if (file) { void initMediaPlayer(file); } });