i think we just fixed audio in the playback forever
This commit is contained in:
Maze Winther
parent
bac5dc97dd
commit
7de8e4172f
|
|
@ -112,6 +112,7 @@ index.ts
|
|||
media: MediaManager
|
||||
renderer: RendererManager
|
||||
save: SaveManager
|
||||
audio: AudioManager
|
||||
static getInstance(): EditorCore
|
||||
static reset(): void
|
||||
}
|
||||
|
|
@ -227,18 +228,18 @@ use-snap-indicator-position.ts
|
|||
}: UseSnapIndicatorPositionParams): SnapIndicatorPosition
|
||||
|
||||
use-timeline-drag-drop.ts
|
||||
export function useTimelineDragDrop({
|
||||
containerRef,
|
||||
zoomLevel,
|
||||
export function useTimelineDragDrop({
|
||||
containerRef,
|
||||
zoomLevel,
|
||||
}: UseTimelineDragDropProps)
|
||||
|
||||
use-timeline-playhead.ts
|
||||
export function useTimelinePlayhead({
|
||||
zoomLevel,
|
||||
rulerRef,
|
||||
rulerScrollRef,
|
||||
tracksScrollRef,
|
||||
playheadRef,
|
||||
export function useTimelinePlayhead({
|
||||
zoomLevel,
|
||||
rulerRef,
|
||||
rulerScrollRef,
|
||||
tracksScrollRef,
|
||||
playheadRef,
|
||||
}: UseTimelinePlayheadProps)
|
||||
|
||||
use-timeline-seek.ts
|
||||
|
|
@ -306,12 +307,12 @@ use-element-resize.ts
|
|||
initialStartTime: number
|
||||
initialDuration: number
|
||||
}
|
||||
export function useTimelineElementResize({
|
||||
element,
|
||||
track,
|
||||
zoomLevel,
|
||||
onSnapPointChange,
|
||||
onResizeStateChange,
|
||||
export function useTimelineElementResize({
|
||||
element,
|
||||
track,
|
||||
zoomLevel,
|
||||
onSnapPointChange,
|
||||
onResizeStateChange,
|
||||
}: UseTimelineElementResizeProps)
|
||||
|
||||
use-element-selection.ts
|
||||
|
|
@ -676,6 +677,16 @@ audio.ts
|
|||
mediaAssets: MediaAsset[];
|
||||
audioContext: AudioContext;
|
||||
}): Promise<CollectedAudioElement[]>
|
||||
export interface AudioClipSource {
|
||||
id: string
|
||||
sourceKey: string
|
||||
file: File
|
||||
startTime: number
|
||||
duration: number
|
||||
trimStart: number
|
||||
trimEnd: number
|
||||
muted: boolean
|
||||
}
|
||||
export function collectAudioMixSources({
|
||||
tracks,
|
||||
mediaAssets,
|
||||
|
|
@ -683,16 +694,25 @@ audio.ts
|
|||
tracks: TimelineTrack[];
|
||||
mediaAssets: MediaAsset[];
|
||||
}): Promise<AudioMixSource[]>
|
||||
export function collectAudioClips({
|
||||
tracks,
|
||||
mediaAssets,
|
||||
}: {
|
||||
tracks: TimelineTrack[];
|
||||
mediaAssets: MediaAsset[];
|
||||
}): Promise<AudioClipSource[]>
|
||||
export function createTimelineAudioBuffer({
|
||||
tracks,
|
||||
mediaAssets,
|
||||
duration,
|
||||
sampleRate = 44100,
|
||||
audioContext,
|
||||
}: {
|
||||
tracks: TimelineTrack[];
|
||||
mediaAssets: MediaAsset[];
|
||||
duration: number;
|
||||
sampleRate?: number;
|
||||
audioContext?: AudioContext;
|
||||
}): Promise<AudioBuffer | null>
|
||||
|
||||
media-utils.ts
|
||||
|
|
@ -1047,21 +1067,29 @@ scene-builder.ts
|
|||
scene-exporter.ts
|
||||
export type ExportFormat = "mp4" | "webm"
|
||||
export type ExportQuality = "low" | "medium" | "high" | "very_high"
|
||||
export type SceneExporterEvents = {
|
||||
progress: [progress: number];
|
||||
complete: [buffer: ArrayBuffer];
|
||||
error: [error: Error];
|
||||
c...
|
||||
export type SceneExporterEvents = {
|
||||
progress: [progress: number];
|
||||
complete: [buffer: ArrayBuffer];
|
||||
error: [error: Error];
|
||||
cance...
|
||||
export class SceneExporter extends EventEmitter<SceneExporterEvents> {
|
||||
renderer: CanvasRenderer
|
||||
format: ExportFormat
|
||||
quality: ExportQuality
|
||||
includeAudio: boolean
|
||||
shouldIncludeAudio: boolean
|
||||
audioBuffer: AudioBuffer
|
||||
cancelled
|
||||
constructor(params: ExportParams)
|
||||
cancel()
|
||||
async export(rootNode: RootNode)
|
||||
isCancelled
|
||||
constructor({
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
format,
|
||||
quality,
|
||||
shouldIncludeAudio,
|
||||
audioBuffer,
|
||||
}: ExportParams)
|
||||
cancel(): void
|
||||
async export({ rootNode }: { rootNode: RootNode }): Promise<ArrayBuffer | null>
|
||||
}
|
||||
|
||||
## apps/web/src/services/storage
|
||||
|
|
@ -1669,9 +1697,6 @@ brand.tsx
|
|||
size?: number;
|
||||
})
|
||||
|
||||
create-icon.tsx
|
||||
export function createIcon({ definition }: CreateIconParams)
|
||||
|
||||
editor.tsx
|
||||
export function OcBackgroundIcon({ className }: { className?: string })
|
||||
export function OcSocialsIcon({
|
||||
|
|
@ -1689,22 +1714,6 @@ editor.tsx
|
|||
size?: number;
|
||||
})
|
||||
|
||||
types.ts
|
||||
export type IconProps = Omit<
|
||||
SVGProps<SVGSVGElement>,
|
||||
"children" | "width" | "height"
|
||||
> & {
|
||||
size?: number;
|
||||
title?: s...
|
||||
export type IconNode = Array<{
|
||||
element: keyof JSX.IntrinsicElements;
|
||||
props: Record<string, string | number | undefined...
|
||||
export type IconDefinition = {
|
||||
title: string;
|
||||
viewBox?: string;
|
||||
nodes: IconNode;
|
||||
}
|
||||
|
||||
ui.tsx
|
||||
export function OcMenuIcon({
|
||||
className = "",
|
||||
|
|
@ -1735,17 +1744,6 @@ ui.tsx
|
|||
size?: number;
|
||||
})
|
||||
|
||||
## packages/ui/src/icons/registry
|
||||
|
||||
editor.ts
|
||||
export const EDITOR_ICONS
|
||||
|
||||
index.ts
|
||||
export const ICON_DEFINITIONS
|
||||
|
||||
ui.ts
|
||||
export const UI_ICONS
|
||||
|
||||
```
|
||||
|
||||
---
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { MediaManager } from "./managers/media-manager";
|
|||
import { RendererManager } from "./managers/renderer-manager";
|
||||
import { CommandManager } from "./managers/commands";
|
||||
import { SaveManager } from "./managers/save-manager";
|
||||
import { AudioManager } from "./managers/audio-manager";
|
||||
|
||||
export class EditorCore {
|
||||
private static instance: EditorCore | null = null;
|
||||
|
|
@ -18,6 +19,7 @@ export class EditorCore {
|
|||
public readonly media: MediaManager;
|
||||
public readonly renderer: RendererManager;
|
||||
public readonly save: SaveManager;
|
||||
public readonly audio: AudioManager;
|
||||
|
||||
private constructor() {
|
||||
this.command = new CommandManager();
|
||||
|
|
@ -28,6 +30,7 @@ export class EditorCore {
|
|||
this.media = new MediaManager(this);
|
||||
this.renderer = new RendererManager(this);
|
||||
this.save = new SaveManager(this);
|
||||
this.audio = new AudioManager(this);
|
||||
this.save.start();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,335 @@
|
|||
import type { EditorCore } from "@/core";
|
||||
import type { AudioClipSource } from "@/lib/media/audio";
|
||||
import { createAudioContext, collectAudioClips } from "@/lib/media/audio";
|
||||
import {
|
||||
ALL_FORMATS,
|
||||
AudioBufferSink,
|
||||
BlobSource,
|
||||
Input,
|
||||
type WrappedAudioBuffer,
|
||||
} from "mediabunny";
|
||||
|
||||
export class AudioManager {
|
||||
private audioContext: AudioContext | null = null;
|
||||
private masterGain: GainNode | null = null;
|
||||
private playbackStartTime = 0;
|
||||
private playbackStartContextTime = 0;
|
||||
private scheduleTimer: number | null = null;
|
||||
private lookaheadSeconds = 2;
|
||||
private scheduleIntervalMs = 500;
|
||||
private clips: AudioClipSource[] = [];
|
||||
private sinks = new Map<string, AudioBufferSink>();
|
||||
private inputs = new Map<string, Input>();
|
||||
private activeClipIds = new Set<string>();
|
||||
private clipIterators = new Map<
|
||||
string,
|
||||
AsyncGenerator<WrappedAudioBuffer, void, unknown>
|
||||
>();
|
||||
private queuedSources = new Set<AudioBufferSourceNode>();
|
||||
private playbackSessionId = 0;
|
||||
private lastIsPlaying = false;
|
||||
private lastVolume = 1;
|
||||
private unsubscribers: Array<() => void> = [];
|
||||
|
||||
constructor(private editor: EditorCore) {
|
||||
this.lastVolume = this.editor.playback.getVolume();
|
||||
|
||||
this.unsubscribers.push(
|
||||
this.editor.playback.subscribe(this.handlePlaybackChange),
|
||||
this.editor.timeline.subscribe(this.handleTimelineChange),
|
||||
this.editor.media.subscribe(this.handleTimelineChange),
|
||||
);
|
||||
if (typeof window !== "undefined") {
|
||||
window.addEventListener("playback-seek", this.handleSeek);
|
||||
}
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.stopPlayback();
|
||||
for (const unsub of this.unsubscribers) {
|
||||
unsub();
|
||||
}
|
||||
this.unsubscribers = [];
|
||||
if (typeof window !== "undefined") {
|
||||
window.removeEventListener("playback-seek", this.handleSeek);
|
||||
}
|
||||
this.disposeSinks();
|
||||
if (this.audioContext) {
|
||||
void this.audioContext.close();
|
||||
this.audioContext = null;
|
||||
this.masterGain = null;
|
||||
}
|
||||
}
|
||||
|
||||
private handlePlaybackChange = (): void => {
|
||||
const isPlaying = this.editor.playback.getIsPlaying();
|
||||
const volume = this.editor.playback.getVolume();
|
||||
|
||||
if (volume !== this.lastVolume) {
|
||||
this.lastVolume = volume;
|
||||
this.updateGain();
|
||||
}
|
||||
|
||||
if (isPlaying !== this.lastIsPlaying) {
|
||||
this.lastIsPlaying = isPlaying;
|
||||
if (isPlaying) {
|
||||
void this.startPlayback({
|
||||
time: this.editor.playback.getCurrentTime(),
|
||||
});
|
||||
} else {
|
||||
this.stopPlayback();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
private handleSeek = (event: Event): void => {
|
||||
const detail = (event as CustomEvent<{ time: number }>).detail;
|
||||
if (!detail) return;
|
||||
|
||||
if (this.editor.playback.getIsPlaying()) {
|
||||
void this.startPlayback({ time: detail.time });
|
||||
return;
|
||||
}
|
||||
|
||||
this.stopPlayback();
|
||||
};
|
||||
|
||||
private handleTimelineChange = (): void => {
|
||||
this.disposeSinks();
|
||||
|
||||
if (!this.editor.playback.getIsPlaying()) return;
|
||||
|
||||
void this.startPlayback({ time: this.editor.playback.getCurrentTime() });
|
||||
};
|
||||
|
||||
private ensureAudioContext(): AudioContext | null {
|
||||
if (this.audioContext) return this.audioContext;
|
||||
if (typeof window === "undefined") return null;
|
||||
|
||||
this.audioContext = createAudioContext();
|
||||
this.masterGain = this.audioContext.createGain();
|
||||
this.masterGain.gain.value = this.lastVolume;
|
||||
this.masterGain.connect(this.audioContext.destination);
|
||||
return this.audioContext;
|
||||
}
|
||||
|
||||
private updateGain(): void {
|
||||
if (!this.masterGain) return;
|
||||
this.masterGain.gain.value = this.lastVolume;
|
||||
}
|
||||
|
||||
private getPlaybackTime(): number {
|
||||
if (!this.audioContext) return this.playbackStartTime;
|
||||
const elapsed = this.audioContext.currentTime - this.playbackStartContextTime;
|
||||
return this.playbackStartTime + elapsed;
|
||||
}
|
||||
|
||||
private async startPlayback({ time }: { time: number }): Promise<void> {
|
||||
const audioContext = this.ensureAudioContext();
|
||||
if (!audioContext) return;
|
||||
|
||||
this.stopPlayback();
|
||||
this.playbackSessionId++;
|
||||
|
||||
const tracks = this.editor.timeline.getTracks();
|
||||
const mediaAssets = this.editor.media.getAssets();
|
||||
const duration = this.editor.timeline.getTotalDuration();
|
||||
|
||||
if (duration <= 0) return;
|
||||
|
||||
if (audioContext.state === "suspended") {
|
||||
await audioContext.resume();
|
||||
}
|
||||
|
||||
this.clips = await collectAudioClips({ tracks, mediaAssets });
|
||||
if (!this.editor.playback.getIsPlaying()) return;
|
||||
|
||||
this.playbackStartTime = time;
|
||||
this.playbackStartContextTime = audioContext.currentTime;
|
||||
|
||||
this.scheduleUpcomingClips();
|
||||
|
||||
if (typeof window !== "undefined") {
|
||||
this.scheduleTimer = window.setInterval(() => {
|
||||
this.scheduleUpcomingClips();
|
||||
}, this.scheduleIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleUpcomingClips(): void {
|
||||
if (!this.editor.playback.getIsPlaying()) return;
|
||||
|
||||
const currentTime = this.getPlaybackTime();
|
||||
const windowEnd = currentTime + this.lookaheadSeconds;
|
||||
|
||||
for (const clip of this.clips) {
|
||||
if (clip.muted) continue;
|
||||
if (this.activeClipIds.has(clip.id)) continue;
|
||||
|
||||
const clipEnd = clip.startTime + clip.duration;
|
||||
if (clipEnd <= currentTime) continue;
|
||||
if (clip.startTime > windowEnd) continue;
|
||||
|
||||
this.activeClipIds.add(clip.id);
|
||||
void this.runClipIterator({ clip, startTime: currentTime, sessionId: this.playbackSessionId });
|
||||
}
|
||||
}
|
||||
|
||||
private stopPlayback(): void {
|
||||
if (this.scheduleTimer && typeof window !== "undefined") {
|
||||
window.clearInterval(this.scheduleTimer);
|
||||
}
|
||||
this.scheduleTimer = null;
|
||||
|
||||
for (const iterator of this.clipIterators.values()) {
|
||||
void iterator.return();
|
||||
}
|
||||
this.clipIterators.clear();
|
||||
this.activeClipIds.clear();
|
||||
|
||||
for (const source of this.queuedSources) {
|
||||
try {
|
||||
source.stop();
|
||||
} catch {}
|
||||
source.disconnect();
|
||||
}
|
||||
this.queuedSources.clear();
|
||||
}
|
||||
|
||||
private async runClipIterator({
|
||||
clip,
|
||||
startTime,
|
||||
sessionId,
|
||||
}: {
|
||||
clip: AudioClipSource;
|
||||
startTime: number;
|
||||
sessionId: number;
|
||||
}): Promise<void> {
|
||||
const audioContext = this.ensureAudioContext();
|
||||
if (!audioContext) return;
|
||||
|
||||
const sink = await this.getAudioSink({ clip });
|
||||
if (!sink || !this.editor.playback.getIsPlaying()) return;
|
||||
if (sessionId !== this.playbackSessionId) return;
|
||||
|
||||
const clipStart = clip.startTime;
|
||||
const clipEnd = clip.startTime + clip.duration;
|
||||
|
||||
const iteratorStartTime = Math.max(startTime, clipStart);
|
||||
const sourceStartTime =
|
||||
clip.trimStart + (iteratorStartTime - clip.startTime);
|
||||
|
||||
const iterator = sink.buffers(sourceStartTime);
|
||||
this.clipIterators.set(clip.id, iterator);
|
||||
|
||||
for await (const { buffer, timestamp } of iterator) {
|
||||
if (!this.editor.playback.getIsPlaying()) return;
|
||||
if (sessionId !== this.playbackSessionId) return;
|
||||
|
||||
const timelineTime = clip.startTime + (timestamp - clip.trimStart);
|
||||
if (timelineTime >= clipEnd) break;
|
||||
|
||||
const node = audioContext.createBufferSource();
|
||||
node.buffer = buffer;
|
||||
node.connect(this.masterGain ?? audioContext.destination);
|
||||
|
||||
const startTimestamp =
|
||||
this.playbackStartContextTime +
|
||||
(timelineTime - this.playbackStartTime);
|
||||
|
||||
if (startTimestamp >= audioContext.currentTime) {
|
||||
node.start(startTimestamp);
|
||||
} else {
|
||||
const offset = audioContext.currentTime - startTimestamp;
|
||||
if (offset < buffer.duration) {
|
||||
node.start(audioContext.currentTime, offset);
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
this.queuedSources.add(node);
|
||||
node.addEventListener("ended", () => {
|
||||
node.disconnect();
|
||||
this.queuedSources.delete(node);
|
||||
});
|
||||
|
||||
const aheadTime = timelineTime - this.getPlaybackTime();
|
||||
if (aheadTime >= 1) {
|
||||
await this.waitUntilCaughtUp({ timelineTime, targetAhead: 1 });
|
||||
if (sessionId !== this.playbackSessionId) return;
|
||||
}
|
||||
}
|
||||
|
||||
this.clipIterators.delete(clip.id);
|
||||
// don't remove from activeClipIds - prevents scheduler from restarting this clip
|
||||
// the set is cleared on stopPlayback anyway
|
||||
}
|
||||
|
||||
private waitUntilCaughtUp({
|
||||
timelineTime,
|
||||
targetAhead,
|
||||
}: {
|
||||
timelineTime: number;
|
||||
targetAhead: number;
|
||||
}): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
const checkInterval = setInterval(() => {
|
||||
if (!this.editor.playback.getIsPlaying()) {
|
||||
clearInterval(checkInterval);
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
const playbackTime = this.getPlaybackTime();
|
||||
if (timelineTime - playbackTime < targetAhead) {
|
||||
clearInterval(checkInterval);
|
||||
resolve();
|
||||
}
|
||||
}, 100);
|
||||
});
|
||||
}
|
||||
|
||||
private disposeSinks(): void {
|
||||
for (const iterator of this.clipIterators.values()) {
|
||||
void iterator.return();
|
||||
}
|
||||
this.clipIterators.clear();
|
||||
this.activeClipIds.clear();
|
||||
|
||||
for (const input of this.inputs.values()) {
|
||||
input.dispose();
|
||||
}
|
||||
this.inputs.clear();
|
||||
this.sinks.clear();
|
||||
}
|
||||
|
||||
private async getAudioSink({
|
||||
clip,
|
||||
}: {
|
||||
clip: AudioClipSource;
|
||||
}): Promise<AudioBufferSink | null> {
|
||||
const existingSink = this.sinks.get(clip.sourceKey);
|
||||
if (existingSink) return existingSink;
|
||||
|
||||
try {
|
||||
const input = new Input({
|
||||
source: new BlobSource(clip.file),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
const audioTrack = await input.getPrimaryAudioTrack();
|
||||
if (!audioTrack) {
|
||||
input.dispose();
|
||||
return null;
|
||||
}
|
||||
|
||||
const sink = new AudioBufferSink(audioTrack);
|
||||
this.inputs.set(clip.sourceKey, input);
|
||||
this.sinks.set(clip.sourceKey, sink);
|
||||
return sink;
|
||||
} catch (error) {
|
||||
console.warn("Failed to initialize audio sink:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -6,7 +6,6 @@ export class PlaybackManager {
|
|||
private volume = 1;
|
||||
private muted = false;
|
||||
private previousVolume = 1;
|
||||
private speed = 1.0;
|
||||
private listeners = new Set<() => void>();
|
||||
private playbackTimer: number | null = null;
|
||||
private lastUpdate = 0;
|
||||
|
|
@ -68,17 +67,6 @@ export class PlaybackManager {
|
|||
this.notify();
|
||||
}
|
||||
|
||||
setSpeed({ speed }: { speed: number }): void {
|
||||
this.speed = Math.max(0.1, Math.min(2.0, speed));
|
||||
this.notify();
|
||||
|
||||
window.dispatchEvent(
|
||||
new CustomEvent("playback-speed", {
|
||||
detail: { speed: this.speed },
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
mute(): void {
|
||||
if (this.volume > 0) {
|
||||
this.previousVolume = this.volume;
|
||||
|
|
@ -118,10 +106,6 @@ export class PlaybackManager {
|
|||
return this.muted;
|
||||
}
|
||||
|
||||
getSpeed(): number {
|
||||
return this.speed;
|
||||
}
|
||||
|
||||
subscribe(listener: () => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
|
|
@ -154,7 +138,7 @@ export class PlaybackManager {
|
|||
const delta = (now - this.lastUpdate) / 1000;
|
||||
this.lastUpdate = now;
|
||||
|
||||
const newTime = this.currentTime + delta * this.speed;
|
||||
const newTime = this.currentTime + delta;
|
||||
const duration = this.editor.timeline.getTotalDuration();
|
||||
|
||||
if (duration > 0 && newTime >= duration) {
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ export class RendererManager {
|
|||
fps: exportFps,
|
||||
format,
|
||||
quality,
|
||||
includeAudio: !!includeAudio,
|
||||
shouldIncludeAudio: !!includeAudio,
|
||||
audioBuffer: audioBuffer || undefined,
|
||||
});
|
||||
|
||||
|
|
@ -91,7 +91,7 @@ export class RendererManager {
|
|||
const cancelInterval = setInterval(checkCancel, 100);
|
||||
|
||||
try {
|
||||
const buffer = await exporter.export(scene);
|
||||
const buffer = await exporter.export({ rootNode: scene });
|
||||
clearInterval(cancelInterval);
|
||||
|
||||
if (cancelled) {
|
||||
|
|
|
|||
|
|
@ -144,6 +144,17 @@ interface AudioMixSource {
|
|||
trimEnd: number;
|
||||
}
|
||||
|
||||
export interface AudioClipSource {
|
||||
id: string;
|
||||
sourceKey: string;
|
||||
file: File;
|
||||
startTime: number;
|
||||
duration: number;
|
||||
trimStart: number;
|
||||
trimEnd: number;
|
||||
muted: boolean;
|
||||
}
|
||||
|
||||
async function fetchLibraryAudioSource({
|
||||
element,
|
||||
}: {
|
||||
|
|
@ -173,6 +184,40 @@ async function fetchLibraryAudioSource({
|
|||
}
|
||||
}
|
||||
|
||||
async function fetchLibraryAudioClip({
|
||||
element,
|
||||
muted,
|
||||
}: {
|
||||
element: LibraryAudioElement;
|
||||
muted: boolean;
|
||||
}): Promise<AudioClipSource | null> {
|
||||
try {
|
||||
const response = await fetch(element.sourceUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Library audio fetch failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
const file = new File([blob], `${element.name}.mp3`, {
|
||||
type: "audio/mpeg",
|
||||
});
|
||||
|
||||
return {
|
||||
id: element.id,
|
||||
sourceKey: element.id,
|
||||
file,
|
||||
startTime: element.startTime,
|
||||
duration: element.duration,
|
||||
trimStart: element.trimStart,
|
||||
trimEnd: element.trimEnd,
|
||||
muted,
|
||||
};
|
||||
} catch (error) {
|
||||
console.warn("Failed to fetch library audio:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function collectMediaAudioSource({
|
||||
element,
|
||||
mediaAsset,
|
||||
|
|
@ -189,6 +234,27 @@ function collectMediaAudioSource({
|
|||
};
|
||||
}
|
||||
|
||||
function collectMediaAudioClip({
|
||||
element,
|
||||
mediaAsset,
|
||||
muted,
|
||||
}: {
|
||||
element: TimelineElement;
|
||||
mediaAsset: MediaAsset;
|
||||
muted: boolean;
|
||||
}): AudioClipSource {
|
||||
return {
|
||||
id: element.id,
|
||||
sourceKey: mediaAsset.id,
|
||||
file: mediaAsset.file,
|
||||
startTime: element.startTime,
|
||||
duration: element.duration,
|
||||
trimStart: element.trimStart,
|
||||
trimEnd: element.trimEnd,
|
||||
muted,
|
||||
};
|
||||
}
|
||||
|
||||
export async function collectAudioMixSources({
|
||||
tracks,
|
||||
mediaAssets,
|
||||
|
|
@ -243,30 +309,98 @@ export async function collectAudioMixSources({
|
|||
return audioMixSources;
|
||||
}
|
||||
|
||||
export async function collectAudioClips({
|
||||
tracks,
|
||||
mediaAssets,
|
||||
}: {
|
||||
tracks: TimelineTrack[];
|
||||
mediaAssets: MediaAsset[];
|
||||
}): Promise<AudioClipSource[]> {
|
||||
const clips: AudioClipSource[] = [];
|
||||
const mediaMap = new Map<string, MediaAsset>(
|
||||
mediaAssets.map((asset) => [asset.id, asset]),
|
||||
);
|
||||
const pendingLibraryClips: Array<Promise<AudioClipSource | null>> = [];
|
||||
|
||||
for (const track of tracks) {
|
||||
const isTrackMuted = canTracktHaveAudio(track) && track.muted;
|
||||
|
||||
for (const element of track.elements) {
|
||||
if (!canElementHaveAudio(element)) continue;
|
||||
|
||||
const isElementMuted =
|
||||
"muted" in element ? (element.muted ?? false) : false;
|
||||
const muted = isTrackMuted || isElementMuted;
|
||||
|
||||
if (element.type === "audio") {
|
||||
if (element.sourceType === "upload") {
|
||||
const mediaAsset = mediaMap.get(element.mediaId);
|
||||
if (!mediaAsset) continue;
|
||||
|
||||
clips.push(
|
||||
collectMediaAudioClip({
|
||||
element,
|
||||
mediaAsset,
|
||||
muted,
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
pendingLibraryClips.push(fetchLibraryAudioClip({ element, muted }));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (element.type === "video") {
|
||||
const mediaAsset = mediaMap.get(element.mediaId);
|
||||
if (!mediaAsset) continue;
|
||||
|
||||
if (mediaSupportsAudio({ media: mediaAsset })) {
|
||||
clips.push(
|
||||
collectMediaAudioClip({
|
||||
element,
|
||||
mediaAsset,
|
||||
muted,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const resolvedLibraryClips = await Promise.all(pendingLibraryClips);
|
||||
for (const clip of resolvedLibraryClips) {
|
||||
if (clip) clips.push(clip);
|
||||
}
|
||||
|
||||
return clips;
|
||||
}
|
||||
|
||||
export async function createTimelineAudioBuffer({
|
||||
tracks,
|
||||
mediaAssets,
|
||||
duration,
|
||||
sampleRate = 44100,
|
||||
audioContext,
|
||||
}: {
|
||||
tracks: TimelineTrack[];
|
||||
mediaAssets: MediaAsset[];
|
||||
duration: number;
|
||||
sampleRate?: number;
|
||||
audioContext?: AudioContext;
|
||||
}): Promise<AudioBuffer | null> {
|
||||
const audioContext = createAudioContext();
|
||||
const context = audioContext ?? createAudioContext();
|
||||
|
||||
const audioElements = await collectAudioElements({
|
||||
tracks,
|
||||
mediaAssets,
|
||||
audioContext,
|
||||
audioContext: context,
|
||||
});
|
||||
|
||||
if (audioElements.length === 0) return null;
|
||||
|
||||
const outputChannels = 2;
|
||||
const outputLength = Math.ceil(duration * sampleRate);
|
||||
const outputBuffer = audioContext.createBuffer(
|
||||
const outputBuffer = context.createBuffer(
|
||||
outputChannels,
|
||||
outputLength,
|
||||
sampleRate,
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ type ExportParams = {
|
|||
fps: number;
|
||||
format: ExportFormat;
|
||||
quality: ExportQuality;
|
||||
includeAudio?: boolean;
|
||||
shouldIncludeAudio?: boolean;
|
||||
audioBuffer?: AudioBuffer;
|
||||
};
|
||||
|
||||
|
|
@ -46,30 +46,38 @@ export class SceneExporter extends EventEmitter<SceneExporterEvents> {
|
|||
private renderer: CanvasRenderer;
|
||||
private format: ExportFormat;
|
||||
private quality: ExportQuality;
|
||||
private includeAudio: boolean;
|
||||
private shouldIncludeAudio: boolean;
|
||||
private audioBuffer?: AudioBuffer;
|
||||
|
||||
private cancelled = false;
|
||||
private isCancelled = false;
|
||||
|
||||
constructor(params: ExportParams) {
|
||||
constructor({
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
format,
|
||||
quality,
|
||||
shouldIncludeAudio,
|
||||
audioBuffer,
|
||||
}: ExportParams) {
|
||||
super();
|
||||
this.renderer = new CanvasRenderer({
|
||||
width: params.width,
|
||||
height: params.height,
|
||||
fps: params.fps,
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
});
|
||||
|
||||
this.format = params.format;
|
||||
this.quality = params.quality;
|
||||
this.includeAudio = params.includeAudio ?? false;
|
||||
this.audioBuffer = params.audioBuffer;
|
||||
this.format = format;
|
||||
this.quality = quality;
|
||||
this.shouldIncludeAudio = shouldIncludeAudio ?? false;
|
||||
this.audioBuffer = audioBuffer;
|
||||
}
|
||||
|
||||
cancel() {
|
||||
this.cancelled = true;
|
||||
cancel(): void {
|
||||
this.isCancelled = true;
|
||||
}
|
||||
|
||||
async export(rootNode: RootNode) {
|
||||
async export({ rootNode }: { rootNode: RootNode }): Promise<ArrayBuffer | null> {
|
||||
const { fps } = this.renderer;
|
||||
const frameCount = Math.ceil(rootNode.duration * fps);
|
||||
|
||||
|
|
@ -88,9 +96,8 @@ export class SceneExporter extends EventEmitter<SceneExporterEvents> {
|
|||
|
||||
output.addVideoTrack(videoSource, { frameRate: fps });
|
||||
|
||||
// Add audio track if requested
|
||||
let audioSource: AudioBufferSource | null = null;
|
||||
if (this.includeAudio && this.audioBuffer) {
|
||||
if (this.shouldIncludeAudio && this.audioBuffer) {
|
||||
audioSource = new AudioBufferSource({
|
||||
codec: this.format === "webm" ? "opus" : "aac",
|
||||
bitrate: qualityMap[this.quality],
|
||||
|
|
@ -100,15 +107,13 @@ export class SceneExporter extends EventEmitter<SceneExporterEvents> {
|
|||
|
||||
await output.start();
|
||||
|
||||
// Add audio data after starting
|
||||
if (audioSource && this.audioBuffer) {
|
||||
await audioSource.add(this.audioBuffer);
|
||||
audioSource.close();
|
||||
}
|
||||
|
||||
// Render video frames
|
||||
for (let i = 0; i < frameCount; i++) {
|
||||
if (this.cancelled) {
|
||||
if (this.isCancelled) {
|
||||
await output.cancel();
|
||||
this.emit("cancelled");
|
||||
return null;
|
||||
|
|
@ -121,7 +126,7 @@ export class SceneExporter extends EventEmitter<SceneExporterEvents> {
|
|||
this.emit("progress", i / frameCount);
|
||||
}
|
||||
|
||||
if (this.cancelled) {
|
||||
if (this.isCancelled) {
|
||||
await output.cancel();
|
||||
this.emit("cancelled");
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -69,6 +69,8 @@ export interface LibraryAudioElement extends BaseAudioElement {
|
|||
sourceUrl: string;
|
||||
}
|
||||
|
||||
export type AudioElement = UploadAudioElement | LibraryAudioElement;
|
||||
|
||||
interface BaseTimelineElement {
|
||||
id: string;
|
||||
name: string;
|
||||
|
|
@ -78,8 +80,6 @@ interface BaseTimelineElement {
|
|||
trimEnd: number;
|
||||
}
|
||||
|
||||
export type AudioElement = UploadAudioElement | LibraryAudioElement;
|
||||
|
||||
export interface VideoElement extends BaseTimelineElement {
|
||||
type: "video";
|
||||
mediaId: string;
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,771 @@
|
|||
import {
|
||||
ALL_FORMATS,
|
||||
AudioBufferSink,
|
||||
BlobSource,
|
||||
CanvasSink,
|
||||
Input,
|
||||
UrlSource,
|
||||
type WrappedAudioBuffer,
|
||||
type WrappedCanvas,
|
||||
} from "mediabunny";
|
||||
|
||||
import SampleFileUrl from "../../docs/assets/big-buck-bunny-trimmed.mp4";
|
||||
(document.querySelector("#sample-file-download") as HTMLAnchorElement).href =
|
||||
SampleFileUrl;
|
||||
|
||||
const selectMediaButton = document.querySelector(
|
||||
"#select-file",
|
||||
) as HTMLButtonElement;
|
||||
const loadUrlButton = document.querySelector("#load-url") as HTMLButtonElement;
|
||||
const fileNameElement = document.querySelector(
|
||||
"#file-name",
|
||||
) as HTMLParagraphElement;
|
||||
const horizontalRule = document.querySelector("hr") as HTMLHRElement;
|
||||
const loadingElement = document.querySelector(
|
||||
"#loading-element",
|
||||
) as HTMLParagraphElement;
|
||||
const playerContainer = document.querySelector("#player") as HTMLDivElement;
|
||||
const canvas = document.querySelector("canvas") as HTMLCanvasElement;
|
||||
const controlsElement = document.querySelector("#controls") as HTMLDivElement;
|
||||
const playButton = document.querySelector("#play-button") as HTMLButtonElement;
|
||||
const playIcon = document.querySelector("#play-icon") as HTMLSpanElement;
|
||||
const pauseIcon = document.querySelector("#pause-icon") as HTMLSpanElement;
|
||||
const currentTimeElement = document.querySelector(
|
||||
"#current-time",
|
||||
) as HTMLSpanElement;
|
||||
const durationElement = document.querySelector("#duration") as HTMLSpanElement;
|
||||
const progressBarContainer = document.querySelector(
|
||||
"#progress-bar-container",
|
||||
) as HTMLDivElement;
|
||||
const progressBar = document.querySelector("#progress-bar") as HTMLDivElement;
|
||||
const volumeBarContainer = document.querySelector(
|
||||
"#volume-bar-container",
|
||||
) as HTMLDivElement;
|
||||
const volumeBar = document.querySelector("#volume-bar") as HTMLDivElement;
|
||||
const volumeIconWrapper = document.querySelector(
|
||||
"#volume-icon-wrapper",
|
||||
) as HTMLDivElement;
|
||||
const volumeButton = document.querySelector(
|
||||
"#volume-button",
|
||||
) as HTMLButtonElement;
|
||||
const fullscreenButton = document.querySelector(
|
||||
"#fullscreen-button",
|
||||
) as HTMLButtonElement;
|
||||
const errorElement = document.querySelector("#error-element") as HTMLDivElement;
|
||||
const warningElement = document.querySelector(
|
||||
"#warning-element",
|
||||
) as HTMLDivElement;
|
||||
|
||||
const context = canvas.getContext("2d")!;
|
||||
|
||||
let audioContext: AudioContext | null = null;
|
||||
let gainNode: GainNode | null = null;
|
||||
|
||||
let fileLoaded = false;
|
||||
let videoSink: CanvasSink | null = null;
|
||||
let audioSink: AudioBufferSink | null = null;
|
||||
|
||||
let totalDuration = 0;
|
||||
/** The value of the audio context's currentTime the moment the playback was started. */
|
||||
let audioContextStartTime: number | null = null;
|
||||
let playing = false;
|
||||
/** The timestamp within the media file when the playback was started. */
|
||||
let playbackTimeAtStart = 0;
|
||||
|
||||
let videoFrameIterator: AsyncGenerator<WrappedCanvas, void, unknown> | null =
|
||||
null;
|
||||
let audioBufferIterator: AsyncGenerator<
|
||||
WrappedAudioBuffer,
|
||||
void,
|
||||
unknown
|
||||
> | null = null;
|
||||
let nextFrame: WrappedCanvas | null = null;
|
||||
const queuedAudioNodes: Set<AudioBufferSourceNode> = new Set();
|
||||
|
||||
/**
|
||||
* Used to prevent async race conditions. When seekId is incremented, already-running async functions will be prevented
|
||||
* from having an effect.
|
||||
*/
|
||||
let asyncId = 0;
|
||||
|
||||
let draggingProgressBar = false;
|
||||
let volume = 0.7;
|
||||
let draggingVolumeBar = false;
|
||||
let volumeMuted = false;
|
||||
|
||||
/** === INIT LOGIC === */
|
||||
|
||||
const initMediaPlayer = async (resource: File | string) => {
|
||||
try {
|
||||
// First, dispose any ongoing playback:
|
||||
|
||||
if (playing) {
|
||||
pause();
|
||||
}
|
||||
|
||||
void videoFrameIterator?.return();
|
||||
void audioBufferIterator?.return();
|
||||
asyncId++;
|
||||
|
||||
fileLoaded = false;
|
||||
fileNameElement.textContent =
|
||||
resource instanceof File ? resource.name : resource;
|
||||
horizontalRule.style.display = "";
|
||||
loadingElement.style.display = "";
|
||||
playerContainer.style.display = "none";
|
||||
errorElement.textContent = "";
|
||||
warningElement.textContent = "";
|
||||
|
||||
// Create an Input from the resource
|
||||
const source =
|
||||
resource instanceof File
|
||||
? new BlobSource(resource)
|
||||
: new UrlSource(resource);
|
||||
const input = new Input({
|
||||
source,
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
playbackTimeAtStart = 0;
|
||||
totalDuration = await input.computeDuration();
|
||||
durationElement.textContent = formatSeconds(totalDuration);
|
||||
|
||||
let videoTrack = await input.getPrimaryVideoTrack();
|
||||
let audioTrack = await input.getPrimaryAudioTrack();
|
||||
|
||||
let problemMessage = "";
|
||||
|
||||
if (videoTrack) {
|
||||
if (videoTrack.codec === null) {
|
||||
problemMessage += "Unsupported video codec. ";
|
||||
videoTrack = null;
|
||||
} else if (!(await videoTrack.canDecode())) {
|
||||
problemMessage += "Unable to decode the video track. ";
|
||||
videoTrack = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (audioTrack) {
|
||||
if (audioTrack.codec === null) {
|
||||
problemMessage += "Unsupported audio codec. ";
|
||||
audioTrack = null;
|
||||
} else if (!(await audioTrack.canDecode())) {
|
||||
problemMessage += "Unable to decode the audio track. ";
|
||||
audioTrack = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (!videoTrack && !audioTrack) {
|
||||
if (!problemMessage) {
|
||||
problemMessage = "No audio or video track found.";
|
||||
}
|
||||
|
||||
throw new Error(problemMessage);
|
||||
}
|
||||
|
||||
if (problemMessage) {
|
||||
warningElement.textContent = problemMessage;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access
|
||||
const AudioContext =
|
||||
window.AudioContext || (window as any).webkitAudioContext;
|
||||
|
||||
// We must create the audio context with the matching sample rate for correct acoustic results
|
||||
// (especially for low-sample rate files)
|
||||
audioContext = new AudioContext({ sampleRate: audioTrack?.sampleRate });
|
||||
gainNode = audioContext.createGain();
|
||||
gainNode.connect(audioContext.destination);
|
||||
updateVolume();
|
||||
|
||||
const videoCanBeTransparent = videoTrack
|
||||
? await videoTrack.canBeTransparent()
|
||||
: false;
|
||||
|
||||
playerContainer.style.background = videoCanBeTransparent
|
||||
? "transparent"
|
||||
: "";
|
||||
|
||||
// For video, let's use a CanvasSink as it handles rotation and closing video samples for us.
|
||||
// Pool size of 2: We'll only ever have the current and the next frame around, so we only need two canvases.
|
||||
videoSink =
|
||||
videoTrack &&
|
||||
new CanvasSink(videoTrack, {
|
||||
poolSize: 2,
|
||||
fit: "contain", // In case the video changes dimensions over time
|
||||
alpha: videoCanBeTransparent,
|
||||
});
|
||||
// For audio, we'll use an AudioBufferSink to directly retrieve AudioBuffers compatible with the Web Audio API
|
||||
audioSink = audioTrack && new AudioBufferSink(audioTrack);
|
||||
|
||||
// Show the canvas if there's a video track, otherwise hide it
|
||||
if (videoTrack) {
|
||||
canvas.style.display = "";
|
||||
canvas.width = videoTrack.displayWidth;
|
||||
canvas.height = videoTrack.displayHeight;
|
||||
} else {
|
||||
canvas.style.display = "none";
|
||||
}
|
||||
|
||||
// Show volume controls if there's an audio track, otherwise hide them
|
||||
if (audioTrack) {
|
||||
volumeButton.style.display = "";
|
||||
volumeBarContainer.style.display = "";
|
||||
} else {
|
||||
volumeButton.style.display = "none";
|
||||
volumeBarContainer.style.display = "none";
|
||||
}
|
||||
|
||||
fileLoaded = true;
|
||||
|
||||
await startVideoIterator();
|
||||
|
||||
if (audioContext.state === "running") {
|
||||
// Start playback automatically if the audio context permits
|
||||
await play();
|
||||
}
|
||||
|
||||
loadingElement.style.display = "none";
|
||||
playerContainer.style.display = "";
|
||||
|
||||
if (!videoSink) {
|
||||
// If there's only an audio track, always show the controls
|
||||
controlsElement.style.opacity = "1";
|
||||
controlsElement.style.pointerEvents = "";
|
||||
playerContainer.style.cursor = "";
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
|
||||
errorElement.textContent = String(error);
|
||||
loadingElement.style.display = "none";
|
||||
playerContainer.style.display = "none";
|
||||
}
|
||||
};
|
||||
|
||||
/** === VIDEO RENDERING LOGIC === */
|
||||
|
||||
/** Creates a new video frame iterator and renders the first video frame. */
|
||||
const startVideoIterator = async () => {
|
||||
if (!videoSink) {
|
||||
return;
|
||||
}
|
||||
|
||||
asyncId++;
|
||||
|
||||
await videoFrameIterator?.return(); // Dispose of the current iterator
|
||||
|
||||
// Create a new iterator
|
||||
videoFrameIterator = videoSink.canvases(getPlaybackTime());
|
||||
|
||||
// Get the first two frames
|
||||
const firstFrame = (await videoFrameIterator.next()).value ?? null;
|
||||
const secondFrame = (await videoFrameIterator.next()).value ?? null;
|
||||
|
||||
nextFrame = secondFrame;
|
||||
|
||||
if (firstFrame) {
|
||||
// Draw the first frame
|
||||
context.clearRect(0, 0, canvas.width, canvas.height);
|
||||
context.drawImage(firstFrame.canvas, 0, 0);
|
||||
}
|
||||
};
|
||||
|
||||
/** Runs every frame; updates the canvas if necessary. */
|
||||
const render = (requestFrame = true) => {
|
||||
if (fileLoaded) {
|
||||
const playbackTime = getPlaybackTime();
|
||||
if (playbackTime >= totalDuration) {
|
||||
// Pause playback once the end is reached
|
||||
pause();
|
||||
playbackTimeAtStart = totalDuration;
|
||||
}
|
||||
|
||||
// Check if the current playback time has caught up to the next frame
|
||||
if (nextFrame && nextFrame.timestamp <= playbackTime) {
|
||||
context.clearRect(0, 0, canvas.width, canvas.height);
|
||||
context.drawImage(nextFrame.canvas, 0, 0);
|
||||
nextFrame = null;
|
||||
|
||||
// Request the next frame
|
||||
void updateNextFrame();
|
||||
}
|
||||
|
||||
if (!draggingProgressBar) {
|
||||
updateProgressBarTime(playbackTime);
|
||||
}
|
||||
}
|
||||
|
||||
if (requestFrame) {
|
||||
requestAnimationFrame(() => render());
|
||||
}
|
||||
};
|
||||
render();
|
||||
|
||||
// Also call the render function on an interval to make sure the video keeps updating even if the tab isn't visible
|
||||
setInterval(() => render(false), 500);
|
||||
|
||||
/** Iterates over the video frame iterator until it finds a video frame in the future. */
|
||||
const updateNextFrame = async () => {
|
||||
const currentAsyncId = asyncId;
|
||||
|
||||
// We have a loop here because we may need to iterate over multiple frames until we reach a frame in the future
|
||||
while (true) {
|
||||
const newNextFrame = (await videoFrameIterator!.next()).value ?? null;
|
||||
if (!newNextFrame) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (currentAsyncId !== asyncId) {
|
||||
break;
|
||||
}
|
||||
|
||||
const playbackTime = getPlaybackTime();
|
||||
if (newNextFrame.timestamp <= playbackTime) {
|
||||
// Draw it immediately
|
||||
context.clearRect(0, 0, canvas.width, canvas.height);
|
||||
context.drawImage(newNextFrame.canvas, 0, 0);
|
||||
} else {
|
||||
// Save it for later
|
||||
nextFrame = newNextFrame;
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/** === AUDIO PLAYBACK LOGIC === */
|
||||
|
||||
/** Loops over the audio buffer iterator, scheduling the audio to be played in the audio context. */
|
||||
const runAudioIterator = async () => {
|
||||
if (!audioSink) {
|
||||
return;
|
||||
}
|
||||
|
||||
// To play back audio, we loop over all audio chunks (typically very short) of the file and play them at the correct
|
||||
// timestamp. The result is a continuous, uninterrupted audio signal.
|
||||
for await (const { buffer, timestamp } of audioBufferIterator!) {
|
||||
const node = audioContext!.createBufferSource();
|
||||
node.buffer = buffer;
|
||||
node.connect(gainNode!);
|
||||
|
||||
const startTimestamp =
|
||||
audioContextStartTime! + timestamp - playbackTimeAtStart;
|
||||
|
||||
// Two cases: Either, the audio starts in the future or in the past
|
||||
if (startTimestamp >= audioContext!.currentTime) {
|
||||
// If the audio starts in the future, easy, we just schedule it
|
||||
node.start(startTimestamp);
|
||||
} else {
|
||||
// If it starts in the past, then let's only play the audible section that remains from here on out
|
||||
node.start(
|
||||
audioContext!.currentTime,
|
||||
audioContext!.currentTime - startTimestamp,
|
||||
);
|
||||
}
|
||||
|
||||
queuedAudioNodes.add(node);
|
||||
node.onended = () => {
|
||||
queuedAudioNodes.delete(node);
|
||||
};
|
||||
|
||||
// If we're more than a second ahead of the current playback time, let's slow down the loop until time has
|
||||
// passed.
|
||||
if (timestamp - getPlaybackTime() >= 1) {
|
||||
await new Promise<void>((resolve) => {
|
||||
const id = setInterval(() => {
|
||||
if (timestamp - getPlaybackTime() < 1) {
|
||||
clearInterval(id);
|
||||
resolve();
|
||||
}
|
||||
}, 100);
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/** === PLAYBACK CONTROL LOGIC === */
|
||||
|
||||
/** Returns the current playback time in the media file. */
|
||||
const getPlaybackTime = () => {
|
||||
if (playing) {
|
||||
// To ensure perfect audio-video sync, we always use the audio context's clock to determine playback time, even
|
||||
// when there is no audio track.
|
||||
return (
|
||||
audioContext!.currentTime - audioContextStartTime! + playbackTimeAtStart
|
||||
);
|
||||
} else {
|
||||
return playbackTimeAtStart;
|
||||
}
|
||||
};
|
||||
|
||||
const play = async () => {
|
||||
if (audioContext!.state === "suspended") {
|
||||
await audioContext!.resume();
|
||||
}
|
||||
|
||||
if (getPlaybackTime() === totalDuration) {
|
||||
// If we're at the end, let's snap back to the start
|
||||
playbackTimeAtStart = 0;
|
||||
await startVideoIterator();
|
||||
}
|
||||
|
||||
audioContextStartTime = audioContext!.currentTime;
|
||||
playing = true;
|
||||
|
||||
if (audioSink) {
|
||||
// Start the audio iterator
|
||||
void audioBufferIterator?.return();
|
||||
audioBufferIterator = audioSink?.buffers(getPlaybackTime());
|
||||
void runAudioIterator();
|
||||
}
|
||||
|
||||
playIcon.style.display = "none";
|
||||
pauseIcon.style.display = "";
|
||||
};
|
||||
|
||||
const pause = () => {
|
||||
playbackTimeAtStart = getPlaybackTime();
|
||||
playing = false;
|
||||
void audioBufferIterator?.return(); // This stops any for-loops that are iterating the iterator
|
||||
audioBufferIterator = null;
|
||||
|
||||
// Stop all audio nodes that were already queued to play
|
||||
for (const node of queuedAudioNodes) {
|
||||
node.stop();
|
||||
}
|
||||
queuedAudioNodes.clear();
|
||||
|
||||
playIcon.style.display = "";
|
||||
pauseIcon.style.display = "none";
|
||||
};
|
||||
|
||||
const togglePlay = () => {
|
||||
if (playing) {
|
||||
pause();
|
||||
} else {
|
||||
void play();
|
||||
}
|
||||
};
|
||||
|
||||
const seekToTime = async (seconds: number) => {
|
||||
updateProgressBarTime(seconds);
|
||||
|
||||
const wasPlaying = playing;
|
||||
|
||||
if (wasPlaying) {
|
||||
pause();
|
||||
}
|
||||
|
||||
playbackTimeAtStart = seconds;
|
||||
|
||||
await startVideoIterator();
|
||||
|
||||
if (wasPlaying && playbackTimeAtStart < totalDuration) {
|
||||
void play();
|
||||
}
|
||||
};
|
||||
|
||||
/** === PROGRESS BAR LOGIC === */
|
||||
|
||||
const updateProgressBarTime = (seconds: number) => {
|
||||
currentTimeElement.textContent = formatSeconds(seconds);
|
||||
progressBar.style.width = `${(seconds / totalDuration) * 100}%`;
|
||||
};
|
||||
|
||||
progressBarContainer.addEventListener("pointerdown", (event) => {
|
||||
draggingProgressBar = true;
|
||||
progressBarContainer.setPointerCapture(event.pointerId);
|
||||
|
||||
const rect = progressBarContainer.getBoundingClientRect();
|
||||
const completion = Math.max(
|
||||
Math.min((event.clientX - rect.left) / rect.width, 1),
|
||||
0,
|
||||
);
|
||||
updateProgressBarTime(completion * totalDuration);
|
||||
|
||||
clearTimeout(hideControlsTimeout);
|
||||
|
||||
window.addEventListener(
|
||||
"pointerup",
|
||||
(event) => {
|
||||
draggingProgressBar = false;
|
||||
progressBarContainer.releasePointerCapture(event.pointerId);
|
||||
|
||||
const rect = progressBarContainer.getBoundingClientRect();
|
||||
const completion = Math.max(
|
||||
Math.min((event.clientX - rect.left) / rect.width, 1),
|
||||
0,
|
||||
);
|
||||
const newTime = completion * totalDuration;
|
||||
|
||||
void seekToTime(newTime);
|
||||
showControlsTemporarily();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
|
||||
progressBarContainer.addEventListener("pointermove", (event) => {
|
||||
if (draggingProgressBar) {
|
||||
const rect = progressBarContainer.getBoundingClientRect();
|
||||
const completion = Math.max(
|
||||
Math.min((event.clientX - rect.left) / rect.width, 1),
|
||||
0,
|
||||
);
|
||||
updateProgressBarTime(completion * totalDuration);
|
||||
}
|
||||
});
|
||||
|
||||
/** === VOLUME CONTROL LOGIC === */
|
||||
|
||||
const updateVolume = () => {
|
||||
const actualVolume = volumeMuted ? 0 : volume;
|
||||
|
||||
volumeBar.style.width = `${actualVolume * 100}%`;
|
||||
gainNode!.gain.value = actualVolume ** 2; // Quadratic for more fine-grained control
|
||||
|
||||
const iconNumber = volumeMuted ? 0 : Math.ceil(1 + 3 * volume);
|
||||
for (let i = 0; i < volumeIconWrapper.children.length; i++) {
|
||||
const icon = volumeIconWrapper.children[i] as HTMLImageElement;
|
||||
icon.style.display = i === iconNumber ? "" : "none";
|
||||
}
|
||||
};
|
||||
|
||||
volumeBarContainer.addEventListener("pointerdown", (event) => {
|
||||
draggingVolumeBar = true;
|
||||
volumeBarContainer.setPointerCapture(event.pointerId);
|
||||
|
||||
const rect = volumeBarContainer.getBoundingClientRect();
|
||||
volume = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0);
|
||||
volumeMuted = false;
|
||||
updateVolume();
|
||||
|
||||
clearTimeout(hideControlsTimeout);
|
||||
|
||||
window.addEventListener(
|
||||
"pointerup",
|
||||
(event) => {
|
||||
draggingVolumeBar = false;
|
||||
volumeBarContainer.releasePointerCapture(event.pointerId);
|
||||
|
||||
const rect = volumeBarContainer.getBoundingClientRect();
|
||||
volume = Math.max(
|
||||
Math.min((event.clientX - rect.left) / rect.width, 1),
|
||||
0,
|
||||
);
|
||||
updateVolume();
|
||||
|
||||
showControlsTemporarily();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
|
||||
volumeButton.addEventListener("click", () => {
|
||||
volumeMuted = !volumeMuted;
|
||||
updateVolume();
|
||||
});
|
||||
|
||||
volumeBarContainer.addEventListener("pointermove", (event) => {
|
||||
if (draggingVolumeBar) {
|
||||
const rect = volumeBarContainer.getBoundingClientRect();
|
||||
volume = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0);
|
||||
updateVolume();
|
||||
}
|
||||
});
|
||||
|
||||
/** === CONTROL UI LOGIC === */
|
||||
|
||||
const showControlsTemporarily = () => {
|
||||
if (!videoSink) {
|
||||
// Shouldn't run if there's only an audio track
|
||||
return;
|
||||
}
|
||||
|
||||
controlsElement.style.opacity = "1";
|
||||
controlsElement.style.pointerEvents = "";
|
||||
playerContainer.style.cursor = "";
|
||||
|
||||
clearTimeout(hideControlsTimeout);
|
||||
hideControlsTimeout = window.setTimeout(() => {
|
||||
if (draggingProgressBar) {
|
||||
return;
|
||||
}
|
||||
|
||||
hideControls();
|
||||
playerContainer.style.cursor = "none";
|
||||
}, 2000);
|
||||
};
|
||||
|
||||
const hideControls = () => {
|
||||
controlsElement.style.opacity = "0";
|
||||
controlsElement.style.pointerEvents = "none";
|
||||
};
|
||||
hideControls();
|
||||
|
||||
let hideControlsTimeout = -1;
|
||||
playerContainer.addEventListener("pointermove", (event) => {
|
||||
if (event.pointerType !== "touch") {
|
||||
showControlsTemporarily();
|
||||
}
|
||||
});
|
||||
playerContainer.addEventListener("pointerleave", (event) => {
|
||||
if (!videoSink) {
|
||||
// Shouldn't run if there's only an audio track
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
draggingProgressBar ||
|
||||
draggingVolumeBar ||
|
||||
event.pointerType === "touch"
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
hideControls();
|
||||
clearTimeout(hideControlsTimeout);
|
||||
});
|
||||
|
||||
/** === EVENT LISTENERS === */
|
||||
|
||||
playButton.addEventListener("click", togglePlay);
|
||||
window.addEventListener("keydown", (e) => {
|
||||
if (!fileLoaded) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (e.code === "Space" || e.code === "KeyK") {
|
||||
togglePlay();
|
||||
} else if (e.code === "KeyF") {
|
||||
fullscreenButton.click();
|
||||
} else if (e.code === "ArrowLeft") {
|
||||
const newTime = Math.max(getPlaybackTime() - 5, 0);
|
||||
void seekToTime(newTime);
|
||||
} else if (e.code === "ArrowRight") {
|
||||
const newTime = Math.min(getPlaybackTime() + 5, totalDuration);
|
||||
void seekToTime(newTime);
|
||||
} else if (e.code === "KeyM") {
|
||||
volumeButton.click();
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
|
||||
showControlsTemporarily();
|
||||
e.preventDefault();
|
||||
});
|
||||
|
||||
fullscreenButton.addEventListener("click", () => {
|
||||
if (document.fullscreenElement) {
|
||||
void document.exitFullscreen();
|
||||
} else {
|
||||
playerContainer.requestFullscreen().catch((e) => {
|
||||
console.error("Failed to enter fullscreen mode:", e);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// I'm sorry for this
|
||||
const isTouchDevice = () => {
|
||||
return "ontouchstart" in window;
|
||||
};
|
||||
|
||||
playerContainer.addEventListener("click", () => {
|
||||
if (isTouchDevice()) {
|
||||
if (controlsElement.style.opacity === "1") {
|
||||
hideControls();
|
||||
} else {
|
||||
showControlsTemporarily();
|
||||
}
|
||||
} else {
|
||||
togglePlay();
|
||||
}
|
||||
});
|
||||
controlsElement.addEventListener("click", (event) => {
|
||||
// Make sure this does NOT toggle play
|
||||
event.stopPropagation();
|
||||
showControlsTemporarily();
|
||||
});
|
||||
|
||||
/** === UTILS === */
|
||||
|
||||
const formatSeconds = (seconds: number) => {
|
||||
const showMilliseconds = window.innerWidth >= 640;
|
||||
|
||||
seconds = Math.round(seconds * 1000) / 1000; // Round to milliseconds
|
||||
|
||||
const hours = Math.floor(seconds / 3600);
|
||||
const minutes = Math.floor((seconds % 3600) / 60);
|
||||
const remainingSeconds = Math.floor(seconds % 60);
|
||||
const millisecs = Math.floor((1000 * seconds) % 1000)
|
||||
.toString()
|
||||
.padStart(3, "0");
|
||||
|
||||
let result: string;
|
||||
if (hours > 0) {
|
||||
result =
|
||||
`${hours}:${minutes.toString().padStart(2, "0")}` +
|
||||
`:${remainingSeconds.toString().padStart(2, "0")}`;
|
||||
} else {
|
||||
result = `${minutes.toString().padStart(2, "0")}:${remainingSeconds.toString().padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
if (showMilliseconds) {
|
||||
result += `.${millisecs}`;
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
window.addEventListener("resize", () => {
|
||||
if (totalDuration) {
|
||||
updateProgressBarTime(getPlaybackTime());
|
||||
durationElement.textContent = formatSeconds(totalDuration);
|
||||
}
|
||||
});
|
||||
|
||||
/** === FILE SELECTION LOGIC === */
|
||||
|
||||
selectMediaButton.addEventListener("click", () => {
|
||||
const fileInput = document.createElement("input");
|
||||
fileInput.type = "file";
|
||||
fileInput.accept =
|
||||
"video/*,video/x-matroska,video/mp2t,.ts,audio/*,audio/aac";
|
||||
fileInput.addEventListener("change", () => {
|
||||
const file = fileInput.files?.[0];
|
||||
if (!file) {
|
||||
return;
|
||||
}
|
||||
|
||||
void initMediaPlayer(file);
|
||||
});
|
||||
|
||||
fileInput.click();
|
||||
});
|
||||
|
||||
loadUrlButton.addEventListener("click", () => {
|
||||
const url = prompt(
|
||||
"Please enter a URL of a media file. Note that it must be HTTPS and support cross-origin requests, so have the" +
|
||||
" right CORS headers set.",
|
||||
"https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4",
|
||||
);
|
||||
if (!url) {
|
||||
return;
|
||||
}
|
||||
|
||||
void initMediaPlayer(url);
|
||||
});
|
||||
|
||||
document.addEventListener("dragover", (event) => {
|
||||
event.preventDefault();
|
||||
event.dataTransfer!.dropEffect = "copy";
|
||||
});
|
||||
|
||||
document.addEventListener("drop", (event) => {
|
||||
event.preventDefault();
|
||||
const files = event.dataTransfer?.files;
|
||||
const file = files && files.length > 0 ? files[0] : undefined;
|
||||
if (file) {
|
||||
void initMediaPlayer(file);
|
||||
}
|
||||
});
|
||||
Loading…
Reference in New Issue