feat(media): support browsers without WebCodecs via FFmpeg + HTML5 fallbacks
OpenCut's video preview and audio playback both rely on mediabunny, which in turn requires the WebCodecs API (VideoDecoder / AudioDecoder). On browsers where WebCodecs is not exposed (e.g. Linux Chromium builds without proprietary codec support), import and playback both throw "codec not supported" errors and the editor becomes unusable. This change makes the editor work on those browsers without sacrificing the WebCodecs fast-path elsewhere: - Add a server-side /api/convert-video route that transcodes incoming videos to H.264 baseline 3.1 yuv420p via the system ffmpeg binary. processMediaAssets uses it automatically when readVideoFile reports canDecode=false so the asset stored is always something the browser can play. - Add HTMLVideoElementSink, a CanvasSink-shaped sink backed by a hidden <video> element + canvas.drawImage. VideoCache.initializeSink selects it only when WebCodecs VideoDecoder is unavailable; otherwise it keeps using mediabunny's CanvasSink unchanged. - Add FallbackAudioBufferSink, an AudioBufferSink-shaped sink backed by AudioContext.decodeAudioData (which uses the platform's native audio decoders, not WebCodecs). AudioManager and the import-time decode path swap to it on canDecode=false. runClipIterator is wrapped in a try/catch so a sink failure no longer leaves an unhandled rejection. Tests cover frame timing, dispose semantics, FPS clamping, the WebCodecs/AudioDecoder availability probes, AudioBuffer slicing, and the fallback iterator contract.
This commit is contained in:
Andrés Villagrán
parent
fbe3db74d7
commit
7204146467
|
|
@ -0,0 +1,144 @@
|
|||
import { type NextRequest, NextResponse } from "next/server";
|
||||
import { writeFile, readFile, rm, mkdtemp } from "fs/promises";
|
||||
import { spawn } from "child_process";
|
||||
import path from "path";
|
||||
import os from "os";
|
||||
|
||||
const GENERIC_CONVERSION_ERROR = "Video conversion failed.";
|
||||
|
||||
function sanitizeContentDispositionName({
|
||||
rawName,
|
||||
}: {
|
||||
rawName: string;
|
||||
}): string {
|
||||
// Strip characters that can break the Content-Disposition header (quotes,
|
||||
// backslashes, CR/LF, control bytes) and replace non-ASCII so the
|
||||
// "filename=" parameter stays unambiguous across clients.
|
||||
const stripped = rawName
|
||||
// eslint-disable-next-line no-control-regex
|
||||
.replace(/[\\"\r\n\x00-\x1f]/g, "_")
|
||||
.replace(/[^\x20-\x7e]/g, "_")
|
||||
.slice(0, 200);
|
||||
return stripped.length > 0 ? stripped : "video";
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
let workDir: string | null = null;
|
||||
|
||||
try {
|
||||
const formData = await request.formData();
|
||||
const file = formData.get("file") as File | null;
|
||||
|
||||
if (!file || !(file instanceof File)) {
|
||||
return NextResponse.json(
|
||||
{ error: "No file provided" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
workDir = await mkdtemp(path.join(os.tmpdir(), "opencut-convert-"));
|
||||
const ext = path.extname(file.name) || ".mp4";
|
||||
const inputPath = path.join(workDir, `input${ext}`);
|
||||
const outputPath = path.join(workDir, "output.mp4");
|
||||
|
||||
const buffer = Buffer.from(await file.arrayBuffer());
|
||||
await writeFile(inputPath, buffer);
|
||||
|
||||
await runFfmpeg({
|
||||
inputPath,
|
||||
outputPath,
|
||||
});
|
||||
|
||||
const convertedBuffer = await readFile(outputPath);
|
||||
|
||||
const safeName = sanitizeContentDispositionName({
|
||||
rawName: path.basename(file.name, ext),
|
||||
});
|
||||
|
||||
return new Response(new Uint8Array(convertedBuffer), {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": "video/mp4",
|
||||
"Content-Disposition": `attachment; filename="${safeName}.mp4"`,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Video conversion error:", error);
|
||||
return NextResponse.json(
|
||||
{ error: GENERIC_CONVERSION_ERROR },
|
||||
{ status: 500 },
|
||||
);
|
||||
} finally {
|
||||
if (workDir) {
|
||||
await rm(workDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function runFfmpeg({
|
||||
inputPath,
|
||||
outputPath,
|
||||
}: {
|
||||
inputPath: string;
|
||||
outputPath: string;
|
||||
}): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const proc = spawn("ffmpeg", [
|
||||
"-i",
|
||||
inputPath,
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-profile:v",
|
||||
"baseline",
|
||||
"-level",
|
||||
"3.1",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"23",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-g",
|
||||
"60",
|
||||
"-keyint_min",
|
||||
"60",
|
||||
"-sc_threshold",
|
||||
"0",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
"-tag:v",
|
||||
"avc1",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-ac",
|
||||
"2",
|
||||
"-ar",
|
||||
"48000",
|
||||
"-b:a",
|
||||
"128k",
|
||||
"-y",
|
||||
outputPath,
|
||||
]);
|
||||
|
||||
let stderr = "";
|
||||
proc.stderr.on("data", (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
proc.on("close", (code) => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(
|
||||
new Error(
|
||||
`ffmpeg exited with code ${code}. stderr: ${stderr.slice(-500)}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
proc.on("error", (err) => {
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
@ -20,6 +20,10 @@ import {
|
|||
Input,
|
||||
type WrappedAudioBuffer,
|
||||
} from "mediabunny";
|
||||
import {
|
||||
type AudioBufferSinkLike,
|
||||
FallbackAudioBufferSink,
|
||||
} from "@/services/audio-cache/fallback-audio-buffer-sink";
|
||||
|
||||
export class AudioManager {
|
||||
private audioContext: AudioContext | null = null;
|
||||
|
|
@ -30,7 +34,7 @@ export class AudioManager {
|
|||
private lookaheadSeconds = 2;
|
||||
private scheduleIntervalMs = 500;
|
||||
private clips: AudioClipSource[] = [];
|
||||
private sinks = new Map<string, AudioBufferSink>();
|
||||
private sinks = new Map<string, AudioBufferSinkLike>();
|
||||
private inputs = new Map<string, Input>();
|
||||
private activeClipIds = new Set<string>();
|
||||
private clipIterators = new Map<
|
||||
|
|
@ -267,7 +271,42 @@ export class AudioManager {
|
|||
|
||||
const iterator = sink.buffers(sourceStartTime);
|
||||
this.clipIterators.set(clip.id, iterator);
|
||||
let consecutiveDroppedBufferCount = 0;
|
||||
|
||||
try {
|
||||
await this.consumeClipIterator({
|
||||
clip,
|
||||
clipEnd,
|
||||
iterator,
|
||||
audioContext,
|
||||
sessionId,
|
||||
initialDroppedCount: 0,
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`[AudioManager] clip iterator failed for ${clip.id}; aborting playback for this clip.`,
|
||||
error,
|
||||
);
|
||||
this.clipIterators.delete(clip.id);
|
||||
this.activeClipIds.delete(clip.id);
|
||||
}
|
||||
}
|
||||
|
||||
private async consumeClipIterator({
|
||||
clip,
|
||||
clipEnd,
|
||||
iterator,
|
||||
audioContext,
|
||||
sessionId,
|
||||
initialDroppedCount,
|
||||
}: {
|
||||
clip: AudioClipSource;
|
||||
clipEnd: number;
|
||||
iterator: AsyncGenerator<WrappedAudioBuffer, void, unknown>;
|
||||
audioContext: AudioContext;
|
||||
sessionId: number;
|
||||
initialDroppedCount: number;
|
||||
}): Promise<void> {
|
||||
let consecutiveDroppedBufferCount = initialDroppedCount;
|
||||
|
||||
for await (const { buffer, timestamp } of iterator) {
|
||||
if (!this.editor.playback.getIsPlaying()) return;
|
||||
|
|
@ -605,7 +644,18 @@ export class AudioManager {
|
|||
return null;
|
||||
}
|
||||
|
||||
const sink = new AudioBufferSink(audioTrack);
|
||||
const canDecode = await safeCanDecode({ audioTrack });
|
||||
const sink: AudioBufferSinkLike = canDecode
|
||||
? new AudioBufferSink(audioTrack)
|
||||
: await FallbackAudioBufferSink.create({
|
||||
file: clip.file,
|
||||
audioContext,
|
||||
});
|
||||
if (!canDecode) {
|
||||
console.warn(
|
||||
"[AudioManager] WebCodecs unavailable; decoding clip audio via decodeAudioData.",
|
||||
);
|
||||
}
|
||||
const chunks: AudioBuffer[] = [];
|
||||
let totalSamples = 0;
|
||||
|
||||
|
|
@ -673,12 +723,13 @@ export class AudioManager {
|
|||
clip,
|
||||
}: {
|
||||
clip: AudioClipSource;
|
||||
}): Promise<AudioBufferSink | null> {
|
||||
}): Promise<AudioBufferSinkLike | null> {
|
||||
const existingSink = this.sinks.get(clip.sourceKey);
|
||||
if (existingSink) return existingSink;
|
||||
|
||||
let input: Input | null = null;
|
||||
try {
|
||||
const input = new Input({
|
||||
input = new Input({
|
||||
source: new BlobSource(clip.file),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
|
@ -688,13 +739,47 @@ export class AudioManager {
|
|||
return null;
|
||||
}
|
||||
|
||||
const canDecode = await safeCanDecode({ audioTrack });
|
||||
if (!canDecode) {
|
||||
input.dispose();
|
||||
input = null;
|
||||
const audioContext = this.ensureAudioContext();
|
||||
if (!audioContext) {
|
||||
return null;
|
||||
}
|
||||
const fallback = await FallbackAudioBufferSink.create({
|
||||
file: clip.file,
|
||||
audioContext,
|
||||
});
|
||||
this.sinks.set(clip.sourceKey, fallback);
|
||||
console.warn(
|
||||
"[AudioManager] WebCodecs unavailable; using decodeAudioData fallback for audio clip.",
|
||||
);
|
||||
return fallback;
|
||||
}
|
||||
|
||||
const sink = new AudioBufferSink(audioTrack);
|
||||
this.inputs.set(clip.sourceKey, input);
|
||||
this.sinks.set(clip.sourceKey, sink);
|
||||
return sink;
|
||||
} catch (error) {
|
||||
if (input) {
|
||||
input.dispose();
|
||||
}
|
||||
console.warn("Failed to initialize audio sink:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function safeCanDecode({
|
||||
audioTrack,
|
||||
}: {
|
||||
audioTrack: { canDecode: () => Promise<boolean> };
|
||||
}): Promise<boolean> {
|
||||
try {
|
||||
return await audioTrack.canDecode();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,10 @@ import { canTrackHaveAudio } from "@/timeline";
|
|||
import { mediaSupportsAudio } from "@/media/media-utils";
|
||||
import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/retime";
|
||||
import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny";
|
||||
import {
|
||||
type AudioBufferSinkLike,
|
||||
FallbackAudioBufferSink,
|
||||
} from "@/services/audio-cache/fallback-audio-buffer-sink";
|
||||
import { TICKS_PER_SECOND } from "@/wasm";
|
||||
import {
|
||||
computeRmsBuckets,
|
||||
|
|
@ -271,7 +275,23 @@ async function resolveAudioBufferForAsset({
|
|||
const audioTrack = await input.getPrimaryAudioTrack();
|
||||
if (!audioTrack) return null;
|
||||
|
||||
const sink = new AudioBufferSink(audioTrack);
|
||||
let audioTrackCanDecode = false;
|
||||
try {
|
||||
audioTrackCanDecode = await audioTrack.canDecode();
|
||||
} catch {
|
||||
audioTrackCanDecode = false;
|
||||
}
|
||||
const sink: AudioBufferSinkLike = audioTrackCanDecode
|
||||
? new AudioBufferSink(audioTrack)
|
||||
: await FallbackAudioBufferSink.create({
|
||||
file: asset.file,
|
||||
audioContext,
|
||||
});
|
||||
if (!audioTrackCanDecode) {
|
||||
console.warn(
|
||||
"[audio] WebCodecs unavailable; decoding asset via decodeAudioData fallback.",
|
||||
);
|
||||
}
|
||||
const targetSampleRate = audioContext.sampleRate;
|
||||
|
||||
const chunks: AudioBuffer[] = [];
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
export async function convertVideoToH264({
|
||||
file,
|
||||
}: {
|
||||
file: File;
|
||||
}): Promise<File> {
|
||||
const formData = new FormData();
|
||||
formData.append("file", file);
|
||||
|
||||
const response = await fetch("/api/convert-video", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}));
|
||||
throw new Error(errorData.error || `Conversion failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
const baseName = file.name.replace(/\.[^.]+$/, "");
|
||||
return new File([blob], `${baseName}.mp4`, { type: "video/mp4" });
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ import type { MediaAsset } from "@/media/types";
|
|||
import { readVideoFile } from "./mediabunny";
|
||||
import type { VideoFileData } from "./mediabunny";
|
||||
import { renderThumbnailDataUrl } from "./thumbnail";
|
||||
import { convertVideoToH264 } from "./ffmpeg-convert";
|
||||
|
||||
export interface ProcessedMediaAsset extends Omit<MediaAsset, "id"> {}
|
||||
|
||||
|
|
@ -132,8 +133,49 @@ export async function processMediaAssets({
|
|||
width = result.width;
|
||||
height = result.height;
|
||||
} else if (fileType === "video") {
|
||||
let currentFile = file;
|
||||
let currentUrl = url;
|
||||
try {
|
||||
const videoData = await readVideoFile({ file });
|
||||
let videoData = await readVideoFile({ file: currentFile });
|
||||
|
||||
if (!videoData.canDecode) {
|
||||
toast.info(`Converting ${file.name} to a compatible format…`);
|
||||
const convertedFile = await convertVideoToH264({
|
||||
file: currentFile,
|
||||
});
|
||||
|
||||
const convertedStorageCheck = await storageService.canStoreFile({
|
||||
size: convertedFile.size,
|
||||
});
|
||||
if (!convertedStorageCheck.canStore) {
|
||||
throw new Error(
|
||||
getStorageLimitDescription({
|
||||
fileSize: convertedFile.size,
|
||||
availableBytes: convertedStorageCheck.availableBytes,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
URL.revokeObjectURL(currentUrl);
|
||||
currentFile = convertedFile;
|
||||
currentUrl = URL.createObjectURL(currentFile);
|
||||
videoData = await readVideoFile({ file: currentFile });
|
||||
|
||||
if (!videoData.canDecode) {
|
||||
// Conversion produced a file the browser still can't decode
|
||||
// via mediabunny/WebCodecs. The HTMLVideoElement fallback in
|
||||
// VideoCache will handle preview rendering, so warn but
|
||||
// continue importing.
|
||||
toast.warning(`${file.name} converted, preview may be limited`, {
|
||||
description: getUnsupportedVideoDescription({
|
||||
codec: videoData.codec,
|
||||
}),
|
||||
});
|
||||
} else {
|
||||
toast.success(`${file.name} converted successfully`);
|
||||
}
|
||||
}
|
||||
|
||||
duration = videoData.duration;
|
||||
width = videoData.width;
|
||||
height = videoData.height;
|
||||
|
|
@ -143,13 +185,27 @@ export async function processMediaAssets({
|
|||
hasAudio = videoData.hasAudio;
|
||||
thumbnailUrl = videoData.thumbnailUrl ?? undefined;
|
||||
|
||||
if (!videoData.canDecode) {
|
||||
toast.error(`Can't preview ${file.name}`, {
|
||||
description: getUnsupportedVideoDescription({
|
||||
codec: videoData.codec,
|
||||
}),
|
||||
});
|
||||
processedAssets.push({
|
||||
name: currentFile.name,
|
||||
type: fileType,
|
||||
file: currentFile,
|
||||
url: currentUrl,
|
||||
thumbnailUrl,
|
||||
duration,
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
hasAudio,
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
completed += 1;
|
||||
if (onProgress) {
|
||||
const percent = Math.round((completed / total) * 100);
|
||||
onProgress({ progress: percent });
|
||||
}
|
||||
continue;
|
||||
} catch (error) {
|
||||
const message =
|
||||
error instanceof Error
|
||||
|
|
@ -159,6 +215,8 @@ export async function processMediaAssets({
|
|||
toast.error(`Couldn't process ${file.name}`, {
|
||||
description: message,
|
||||
});
|
||||
URL.revokeObjectURL(currentUrl);
|
||||
continue;
|
||||
}
|
||||
} else if (fileType === "audio") {
|
||||
duration = await getMediaDuration({ file });
|
||||
|
|
|
|||
|
|
@ -0,0 +1,308 @@
|
|||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
test,
|
||||
} from "bun:test";
|
||||
import {
|
||||
FallbackAudioBufferSink,
|
||||
isWebAudioDecoderAvailable,
|
||||
sliceAudioBuffer,
|
||||
} from "../fallback-audio-buffer-sink";
|
||||
|
||||
type AudioBufferLike = {
|
||||
length: number;
|
||||
sampleRate: number;
|
||||
numberOfChannels: number;
|
||||
duration: number;
|
||||
getChannelData: (channel: number) => Float32Array;
|
||||
copyToChannel: (
|
||||
source: Float32Array,
|
||||
channel: number,
|
||||
startInChannel?: number,
|
||||
) => void;
|
||||
};
|
||||
|
||||
function createTestAudioBuffer({
|
||||
durationSeconds = 1,
|
||||
sampleRate = 8,
|
||||
channelCount = 2,
|
||||
}: {
|
||||
durationSeconds?: number;
|
||||
sampleRate?: number;
|
||||
channelCount?: number;
|
||||
} = {}): AudioBufferLike {
|
||||
const length = Math.round(durationSeconds * sampleRate);
|
||||
const channels = Array.from(
|
||||
{ length: channelCount },
|
||||
(_, channelIndex) =>
|
||||
new Float32Array(
|
||||
Array.from({ length }, (_, i) => channelIndex + 1 + i * 0.01),
|
||||
),
|
||||
);
|
||||
return {
|
||||
length,
|
||||
sampleRate,
|
||||
numberOfChannels: channelCount,
|
||||
duration: durationSeconds,
|
||||
getChannelData(channel: number) {
|
||||
return channels[channel];
|
||||
},
|
||||
// eslint-disable-next-line opencut/prefer-object-params -- mirrors the AudioBuffer.copyToChannel Web API signature so this fake behaves like the real one
|
||||
copyToChannel(
|
||||
source: Float32Array,
|
||||
channel: number,
|
||||
startInChannel = 0,
|
||||
) {
|
||||
channels[channel].set(source, startInChannel);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function installFakeAudioBuffer(): { restore: () => void } {
|
||||
const root = globalThis as { AudioBuffer?: unknown };
|
||||
const original = root.AudioBuffer;
|
||||
class FakeAudioBuffer implements AudioBufferLike {
|
||||
length: number;
|
||||
sampleRate: number;
|
||||
numberOfChannels: number;
|
||||
duration: number;
|
||||
private channels: Float32Array[];
|
||||
|
||||
constructor(opts: {
|
||||
length: number;
|
||||
numberOfChannels: number;
|
||||
sampleRate: number;
|
||||
}) {
|
||||
this.length = opts.length;
|
||||
this.sampleRate = opts.sampleRate;
|
||||
this.numberOfChannels = opts.numberOfChannels;
|
||||
this.duration = opts.length / opts.sampleRate;
|
||||
this.channels = Array.from(
|
||||
{ length: opts.numberOfChannels },
|
||||
() => new Float32Array(opts.length),
|
||||
);
|
||||
}
|
||||
|
||||
getChannelData(channel: number): Float32Array {
|
||||
return this.channels[channel];
|
||||
}
|
||||
|
||||
// eslint-disable-next-line opencut/prefer-object-params -- mirrors AudioBuffer.copyToChannel Web API signature
|
||||
copyToChannel(
|
||||
source: Float32Array,
|
||||
channel: number,
|
||||
startInChannel = 0,
|
||||
): void {
|
||||
this.channels[channel].set(source, startInChannel);
|
||||
}
|
||||
}
|
||||
root.AudioBuffer = FakeAudioBuffer;
|
||||
return {
|
||||
restore() {
|
||||
if (original === undefined) {
|
||||
delete root.AudioBuffer;
|
||||
} else {
|
||||
root.AudioBuffer = original;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("isWebAudioDecoderAvailable", () => {
|
||||
const root = globalThis as { AudioDecoder?: unknown };
|
||||
let original: unknown;
|
||||
|
||||
beforeEach(() => {
|
||||
original = root.AudioDecoder;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (original === undefined) {
|
||||
delete root.AudioDecoder;
|
||||
} else {
|
||||
root.AudioDecoder = original;
|
||||
}
|
||||
});
|
||||
|
||||
test("returns false when AudioDecoder undefined", () => {
|
||||
delete root.AudioDecoder;
|
||||
expect(isWebAudioDecoderAvailable()).toBe(false);
|
||||
});
|
||||
|
||||
test("returns true when AudioDecoder defined", () => {
|
||||
root.AudioDecoder = class {};
|
||||
expect(isWebAudioDecoderAvailable()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sliceAudioBuffer", () => {
|
||||
let fakeAudioBuffer: ReturnType<typeof installFakeAudioBuffer>;
|
||||
|
||||
beforeEach(() => {
|
||||
fakeAudioBuffer = installFakeAudioBuffer();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fakeAudioBuffer.restore();
|
||||
});
|
||||
|
||||
test("slices to the requested range", () => {
|
||||
const buffer = createTestAudioBuffer({
|
||||
durationSeconds: 1,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sliced = sliceAudioBuffer({
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
buffer: buffer as unknown as AudioBuffer,
|
||||
start: 0.25,
|
||||
end: 0.75,
|
||||
});
|
||||
expect(sliced.length).toBe(4);
|
||||
expect(sliced.numberOfChannels).toBe(1);
|
||||
expect(sliced.sampleRate).toBe(8);
|
||||
const channel = sliced.getChannelData(0);
|
||||
expect(channel[0]).toBeCloseTo(1 + 0.02, 5);
|
||||
expect(channel[3]).toBeCloseTo(1 + 0.05, 5);
|
||||
});
|
||||
|
||||
test("clamps out-of-range start/end", () => {
|
||||
const buffer = createTestAudioBuffer({
|
||||
durationSeconds: 0.5,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sliced = sliceAudioBuffer({
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
buffer: buffer as unknown as AudioBuffer,
|
||||
start: -1,
|
||||
end: 5,
|
||||
});
|
||||
expect(sliced.length).toBe(buffer.length);
|
||||
});
|
||||
|
||||
test("preserves channel count", () => {
|
||||
const buffer = createTestAudioBuffer({
|
||||
durationSeconds: 1,
|
||||
sampleRate: 8,
|
||||
channelCount: 2,
|
||||
});
|
||||
const sliced = sliceAudioBuffer({
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
buffer: buffer as unknown as AudioBuffer,
|
||||
start: 0,
|
||||
end: 0.5,
|
||||
});
|
||||
expect(sliced.numberOfChannels).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("FallbackAudioBufferSink.buffers", () => {
|
||||
let fakeAudioBuffer: ReturnType<typeof installFakeAudioBuffer>;
|
||||
|
||||
beforeEach(() => {
|
||||
fakeAudioBuffer = installFakeAudioBuffer();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fakeAudioBuffer.restore();
|
||||
});
|
||||
|
||||
test("yields a single buffer covering the full range by default", async () => {
|
||||
const decoded = createTestAudioBuffer({
|
||||
durationSeconds: 1,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sink = new FallbackAudioBufferSink(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
decoded as unknown as AudioBuffer,
|
||||
);
|
||||
const yielded = [];
|
||||
for await (const wrapped of sink.buffers()) {
|
||||
yielded.push(wrapped);
|
||||
}
|
||||
expect(yielded).toHaveLength(1);
|
||||
expect(yielded[0].timestamp).toBe(0);
|
||||
expect(yielded[0].duration).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
test("yields a slice when startTimestamp/endTimestamp provided", async () => {
|
||||
const decoded = createTestAudioBuffer({
|
||||
durationSeconds: 2,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sink = new FallbackAudioBufferSink(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
decoded as unknown as AudioBuffer,
|
||||
);
|
||||
const yielded = [];
|
||||
for await (const wrapped of sink.buffers(0.5, 1.5)) {
|
||||
yielded.push(wrapped);
|
||||
}
|
||||
expect(yielded).toHaveLength(1);
|
||||
expect(yielded[0].timestamp).toBe(0.5);
|
||||
expect(yielded[0].duration).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
test("treats NaN/Infinity startTimestamp as 0", async () => {
|
||||
const decoded = createTestAudioBuffer({
|
||||
durationSeconds: 1,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sink = new FallbackAudioBufferSink(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
decoded as unknown as AudioBuffer,
|
||||
);
|
||||
const yielded = [];
|
||||
for await (const wrapped of sink.buffers(Number.NaN)) {
|
||||
yielded.push(wrapped);
|
||||
}
|
||||
expect(yielded).toHaveLength(1);
|
||||
expect(yielded[0].timestamp).toBe(0);
|
||||
expect(yielded[0].duration).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
test("yields nothing when start >= end (or beyond duration)", async () => {
|
||||
const decoded = createTestAudioBuffer({
|
||||
durationSeconds: 1,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sink = new FallbackAudioBufferSink(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
decoded as unknown as AudioBuffer,
|
||||
);
|
||||
|
||||
const yielded = [];
|
||||
for await (const wrapped of sink.buffers(2)) {
|
||||
yielded.push(wrapped);
|
||||
}
|
||||
expect(yielded).toHaveLength(0);
|
||||
|
||||
const yielded2 = [];
|
||||
for await (const wrapped of sink.buffers(0.4, 0.4)) {
|
||||
yielded2.push(wrapped);
|
||||
}
|
||||
expect(yielded2).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("exposes sampleRate/numberOfChannels/duration getters", () => {
|
||||
const decoded = createTestAudioBuffer({
|
||||
durationSeconds: 3,
|
||||
sampleRate: 16,
|
||||
channelCount: 2,
|
||||
});
|
||||
const sink = new FallbackAudioBufferSink(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
decoded as unknown as AudioBuffer,
|
||||
);
|
||||
expect(sink.sampleRate).toBe(16);
|
||||
expect(sink.numberOfChannels).toBe(2);
|
||||
expect(sink.duration).toBe(3);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
import type { WrappedAudioBuffer } from "mediabunny";
|
||||
|
||||
export interface AudioBufferSinkLike {
|
||||
buffers(
|
||||
startTimestamp?: number,
|
||||
endTimestamp?: number,
|
||||
): AsyncGenerator<WrappedAudioBuffer, void, unknown>;
|
||||
}
|
||||
|
||||
export function isWebAudioDecoderAvailable(): boolean {
|
||||
return (
|
||||
typeof (globalThis as { AudioDecoder?: unknown }).AudioDecoder !==
|
||||
"undefined"
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* AudioBufferSink implementation backed by AudioContext.decodeAudioData.
|
||||
*
|
||||
* Used when mediabunny's AudioSampleSink cannot decode the track because the
|
||||
* environment lacks the WebCodecs AudioDecoder API. AudioContext.decodeAudioData
|
||||
* uses the system's native audio decoders, mirroring the strategy used by
|
||||
* HTMLVideoElementSink for video. The entire compressed audio is decoded
|
||||
* upfront and exposed through a buffers() iterator that yields a single
|
||||
* AudioBuffer slice covering the requested range.
|
||||
*/
|
||||
export class FallbackAudioBufferSink implements AudioBufferSinkLike {
|
||||
constructor(private readonly decoded: AudioBuffer) {}
|
||||
|
||||
static async create({
|
||||
file,
|
||||
audioContext,
|
||||
}: {
|
||||
file: File;
|
||||
audioContext: BaseAudioContext;
|
||||
}): Promise<FallbackAudioBufferSink> {
|
||||
const arrayBuffer = await file.arrayBuffer();
|
||||
const decoded = await decodeArrayBuffer({ arrayBuffer, audioContext });
|
||||
return new FallbackAudioBufferSink(decoded);
|
||||
}
|
||||
|
||||
get sampleRate(): number {
|
||||
return this.decoded.sampleRate;
|
||||
}
|
||||
|
||||
get numberOfChannels(): number {
|
||||
return this.decoded.numberOfChannels;
|
||||
}
|
||||
|
||||
get duration(): number {
|
||||
return this.decoded.duration;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line opencut/prefer-object-params -- mirrors mediabunny's AudioBufferSink.buffers(start, end) so consumers can swap sinks transparently
|
||||
async *buffers(
|
||||
startTimestamp = 0,
|
||||
endTimestamp?: number,
|
||||
): AsyncGenerator<WrappedAudioBuffer, void, unknown> {
|
||||
const totalDuration = this.decoded.duration;
|
||||
const safeStart = Number.isFinite(startTimestamp) ? startTimestamp : 0;
|
||||
const start = Math.max(0, Math.min(safeStart, totalDuration));
|
||||
const end =
|
||||
typeof endTimestamp === "number" && Number.isFinite(endTimestamp)
|
||||
? Math.min(endTimestamp, totalDuration)
|
||||
: totalDuration;
|
||||
if (start >= end) return;
|
||||
|
||||
const sliced = sliceAudioBuffer({
|
||||
buffer: this.decoded,
|
||||
start,
|
||||
end,
|
||||
});
|
||||
|
||||
yield {
|
||||
buffer: sliced,
|
||||
timestamp: start,
|
||||
duration: sliced.duration,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function decodeArrayBuffer({
|
||||
arrayBuffer,
|
||||
audioContext,
|
||||
}: {
|
||||
arrayBuffer: ArrayBuffer;
|
||||
audioContext: BaseAudioContext;
|
||||
}): Promise<AudioBuffer> {
|
||||
return new Promise<AudioBuffer>((resolve, reject) => {
|
||||
audioContext.decodeAudioData(
|
||||
arrayBuffer,
|
||||
(buffer) => resolve(buffer),
|
||||
(err) =>
|
||||
reject(
|
||||
err instanceof Error
|
||||
? err
|
||||
: new Error(`decodeAudioData failed: ${String(err)}`),
|
||||
),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export function sliceAudioBuffer({
|
||||
buffer,
|
||||
start,
|
||||
end,
|
||||
}: {
|
||||
buffer: AudioBuffer;
|
||||
start: number;
|
||||
end: number;
|
||||
}): AudioBuffer {
|
||||
const sampleRate = buffer.sampleRate;
|
||||
const startSample = Math.max(
|
||||
0,
|
||||
Math.min(buffer.length, Math.floor(start * sampleRate)),
|
||||
);
|
||||
const endSample = Math.max(
|
||||
startSample,
|
||||
Math.min(buffer.length, Math.ceil(end * sampleRate)),
|
||||
);
|
||||
const length = Math.max(1, endSample - startSample);
|
||||
|
||||
const target = new AudioBuffer({
|
||||
length,
|
||||
numberOfChannels: Math.max(1, buffer.numberOfChannels),
|
||||
sampleRate,
|
||||
});
|
||||
for (let channel = 0; channel < buffer.numberOfChannels; channel++) {
|
||||
const channelView = buffer
|
||||
.getChannelData(channel)
|
||||
.subarray(startSample, endSample);
|
||||
target.copyToChannel(channelView, channel, 0);
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
|
@ -0,0 +1,259 @@
|
|||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
test,
|
||||
} from "bun:test";
|
||||
import {
|
||||
clampFallbackFps,
|
||||
HTMLVideoElementSink,
|
||||
isWebCodecsAvailable,
|
||||
type VideoElementHandle,
|
||||
} from "../html-video-element-sink";
|
||||
|
||||
type FakeCanvas = {
|
||||
width: number;
|
||||
height: number;
|
||||
getContext: (kind: string) => FakeCanvasContext | null;
|
||||
};
|
||||
|
||||
type FakeCanvasContext = {
|
||||
drawImage: (...args: unknown[]) => void;
|
||||
drawnCount: number;
|
||||
};
|
||||
|
||||
function installFakeDom(): {
|
||||
createdCanvases: FakeCanvas[];
|
||||
restore: () => void;
|
||||
} {
|
||||
const createdCanvases: FakeCanvas[] = [];
|
||||
const originalDocument = (
|
||||
globalThis as { document?: unknown }
|
||||
).document;
|
||||
|
||||
const fakeDocument = {
|
||||
createElement(tag: string) {
|
||||
if (tag !== "canvas") {
|
||||
throw new Error(`unexpected createElement(${tag}) in test`);
|
||||
}
|
||||
const ctx: FakeCanvasContext = {
|
||||
drawnCount: 0,
|
||||
drawImage() {
|
||||
ctx.drawnCount += 1;
|
||||
},
|
||||
};
|
||||
const canvas: FakeCanvas = {
|
||||
width: 0,
|
||||
height: 0,
|
||||
getContext(kind: string) {
|
||||
return kind === "2d" ? ctx : null;
|
||||
},
|
||||
};
|
||||
createdCanvases.push(canvas);
|
||||
return canvas;
|
||||
},
|
||||
};
|
||||
|
||||
(globalThis as { document?: unknown }).document = fakeDocument;
|
||||
|
||||
return {
|
||||
createdCanvases,
|
||||
restore() {
|
||||
(globalThis as { document?: unknown }).document = originalDocument;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createFakeHandle({
|
||||
duration = 2,
|
||||
videoWidth = 640,
|
||||
videoHeight = 360,
|
||||
}: {
|
||||
duration?: number;
|
||||
videoWidth?: number;
|
||||
videoHeight?: number;
|
||||
} = {}): VideoElementHandle & {
|
||||
seekCalls: number[];
|
||||
drawCalls: number;
|
||||
disposed: boolean;
|
||||
} {
|
||||
const handle = {
|
||||
seekCalls: [] as number[],
|
||||
drawCalls: 0,
|
||||
disposed: false,
|
||||
duration,
|
||||
videoWidth,
|
||||
videoHeight,
|
||||
async seek(time: number) {
|
||||
handle.seekCalls.push(time);
|
||||
},
|
||||
drawTo() {
|
||||
handle.drawCalls += 1;
|
||||
},
|
||||
dispose() {
|
||||
handle.disposed = true;
|
||||
},
|
||||
};
|
||||
return handle;
|
||||
}
|
||||
|
||||
describe("clampFallbackFps", () => {
|
||||
test("returns default for undefined/NaN/zero/negative", () => {
|
||||
expect(clampFallbackFps(undefined)).toBe(30);
|
||||
expect(clampFallbackFps(Number.NaN)).toBe(30);
|
||||
expect(clampFallbackFps(0)).toBe(30);
|
||||
expect(clampFallbackFps(-5)).toBe(30);
|
||||
});
|
||||
|
||||
test("rounds and clamps within [1, 60]", () => {
|
||||
expect(clampFallbackFps(24)).toBe(24);
|
||||
expect(clampFallbackFps(29.4)).toBe(29);
|
||||
expect(clampFallbackFps(120)).toBe(60);
|
||||
expect(clampFallbackFps(0.4)).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isWebCodecsAvailable", () => {
|
||||
const root = globalThis as { VideoDecoder?: unknown };
|
||||
let originalVideoDecoder: unknown;
|
||||
|
||||
beforeEach(() => {
|
||||
originalVideoDecoder = root.VideoDecoder;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (originalVideoDecoder === undefined) {
|
||||
delete root.VideoDecoder;
|
||||
} else {
|
||||
root.VideoDecoder = originalVideoDecoder;
|
||||
}
|
||||
});
|
||||
|
||||
test("false when VideoDecoder is undefined", () => {
|
||||
delete root.VideoDecoder;
|
||||
expect(isWebCodecsAvailable()).toBe(false);
|
||||
});
|
||||
|
||||
test("true when VideoDecoder is defined", () => {
|
||||
root.VideoDecoder = class {};
|
||||
expect(isWebCodecsAvailable()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("HTMLVideoElementSink.canvases", () => {
|
||||
let dom: ReturnType<typeof installFakeDom>;
|
||||
|
||||
beforeEach(() => {
|
||||
dom = installFakeDom();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
dom.restore();
|
||||
});
|
||||
|
||||
test("yields frames stepping by 1/fps from startTimestamp", async () => {
|
||||
const handle = createFakeHandle({ duration: 0.25 });
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 10 }); // step = 0.1s
|
||||
|
||||
const frames: Array<{ timestamp: number; duration: number }> = [];
|
||||
for await (const frame of sink.canvases(0)) {
|
||||
frames.push({
|
||||
timestamp: frame.timestamp,
|
||||
duration: frame.duration,
|
||||
});
|
||||
}
|
||||
|
||||
expect(frames).toHaveLength(3);
|
||||
expect(frames[0].timestamp).toBeCloseTo(0, 5);
|
||||
expect(frames[1].timestamp).toBeCloseTo(0.1, 5);
|
||||
expect(frames[2].timestamp).toBeCloseTo(0.2, 5);
|
||||
expect(frames[0].duration).toBeCloseTo(0.1, 5);
|
||||
expect(handle.seekCalls).toEqual([0, 0.1, 0.2]);
|
||||
expect(handle.drawCalls).toBe(3);
|
||||
});
|
||||
|
||||
test("respects endTimestamp", async () => {
|
||||
const handle = createFakeHandle({ duration: 5 });
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 10 });
|
||||
|
||||
const frames: number[] = [];
|
||||
for await (const frame of sink.canvases(0, 0.2)) {
|
||||
frames.push(frame.timestamp);
|
||||
}
|
||||
|
||||
expect(frames).toHaveLength(2);
|
||||
expect(frames[0]).toBeCloseTo(0, 5);
|
||||
expect(frames[1]).toBeCloseTo(0.1, 5);
|
||||
});
|
||||
|
||||
test("dispose stops the iterator early", async () => {
|
||||
const handle = createFakeHandle({ duration: 10 });
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 10 });
|
||||
|
||||
const collected: number[] = [];
|
||||
for await (const frame of sink.canvases(0)) {
|
||||
collected.push(frame.timestamp);
|
||||
if (collected.length === 2) sink.dispose();
|
||||
}
|
||||
|
||||
expect(collected).toHaveLength(2);
|
||||
expect(handle.disposed).toBe(true);
|
||||
});
|
||||
|
||||
test("returns no frames when start >= duration", async () => {
|
||||
const handle = createFakeHandle({ duration: 1 });
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 30 });
|
||||
|
||||
const frames: number[] = [];
|
||||
for await (const frame of sink.canvases(2)) {
|
||||
frames.push(frame.timestamp);
|
||||
}
|
||||
expect(frames).toHaveLength(0);
|
||||
expect(handle.seekCalls).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("aborts cleanly when seek throws", async () => {
|
||||
const handle = createFakeHandle({ duration: 1 });
|
||||
handle.seek = async () => {
|
||||
throw new Error("boom");
|
||||
};
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 30 });
|
||||
|
||||
const frames: number[] = [];
|
||||
for await (const frame of sink.canvases(0)) {
|
||||
frames.push(frame.timestamp);
|
||||
}
|
||||
expect(frames).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("clamps invalid fps passed to constructor to the default", async () => {
|
||||
const handle = createFakeHandle({ duration: 0.05 });
|
||||
// Passing 0 (negative, or NaN) used to leak through to 1/fps and produce
|
||||
// invalid frame stepping. The constructor must reject these and fall back
|
||||
// to the default FPS.
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 0 });
|
||||
const frames = [];
|
||||
for await (const frame of sink.canvases(0)) {
|
||||
frames.push(frame);
|
||||
}
|
||||
expect(frames).toHaveLength(2);
|
||||
expect(frames[0].duration).toBeCloseTo(1 / 30, 5);
|
||||
});
|
||||
|
||||
test("frame canvas matches video dimensions", async () => {
|
||||
const handle = createFakeHandle({
|
||||
duration: 0.02,
|
||||
videoWidth: 1280,
|
||||
videoHeight: 720,
|
||||
});
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 30 });
|
||||
const frames = [];
|
||||
for await (const frame of sink.canvases(0)) {
|
||||
frames.push(frame);
|
||||
}
|
||||
expect(frames).toHaveLength(1);
|
||||
expect(frames[0].canvas.width).toBe(1280);
|
||||
expect(frames[0].canvas.height).toBe(720);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,249 @@
|
|||
import type { WrappedCanvas } from "mediabunny";
|
||||
|
||||
const DEFAULT_FALLBACK_FPS = 30;
|
||||
const MAX_FALLBACK_FPS = 60;
|
||||
const MIN_FALLBACK_FPS = 1;
|
||||
|
||||
export interface FrameSink {
|
||||
canvases(
|
||||
startTimestamp?: number,
|
||||
endTimestamp?: number,
|
||||
): AsyncGenerator<WrappedCanvas, void, unknown>;
|
||||
}
|
||||
|
||||
export interface VideoElementHandle {
|
||||
readonly duration: number;
|
||||
readonly videoWidth: number;
|
||||
readonly videoHeight: number;
|
||||
seek(time: number): Promise<void>;
|
||||
drawTo(target: {
|
||||
canvas: HTMLCanvasElement | OffscreenCanvas;
|
||||
ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
|
||||
}): void;
|
||||
dispose(): void;
|
||||
}
|
||||
|
||||
export function isWebCodecsAvailable(): boolean {
|
||||
return typeof (globalThis as { VideoDecoder?: unknown }).VideoDecoder !==
|
||||
"undefined";
|
||||
}
|
||||
|
||||
export function clampFallbackFps(requested: number | undefined): number {
|
||||
if (
|
||||
requested === undefined ||
|
||||
!Number.isFinite(requested) ||
|
||||
requested <= 0
|
||||
) {
|
||||
return DEFAULT_FALLBACK_FPS;
|
||||
}
|
||||
const rounded = Math.round(requested);
|
||||
if (rounded < MIN_FALLBACK_FPS) return MIN_FALLBACK_FPS;
|
||||
if (rounded > MAX_FALLBACK_FPS) return MAX_FALLBACK_FPS;
|
||||
return rounded;
|
||||
}
|
||||
|
||||
export async function createHTMLVideoElementHandle({
|
||||
file,
|
||||
}: {
|
||||
file: File;
|
||||
}): Promise<VideoElementHandle> {
|
||||
const objectUrl = URL.createObjectURL(file);
|
||||
const video = document.createElement("video");
|
||||
video.preload = "auto";
|
||||
video.muted = true;
|
||||
video.playsInline = true;
|
||||
video.crossOrigin = "anonymous";
|
||||
video.src = objectUrl;
|
||||
|
||||
try {
|
||||
await waitForLoadedMetadata({ video });
|
||||
} catch (error) {
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
throw error;
|
||||
}
|
||||
|
||||
let disposed = false;
|
||||
|
||||
return {
|
||||
get duration() {
|
||||
return video.duration;
|
||||
},
|
||||
get videoWidth() {
|
||||
return video.videoWidth;
|
||||
},
|
||||
get videoHeight() {
|
||||
return video.videoHeight;
|
||||
},
|
||||
async seek(time) {
|
||||
if (disposed) {
|
||||
throw new Error("VideoElementHandle disposed");
|
||||
}
|
||||
await seekVideoElement({ video, time });
|
||||
},
|
||||
drawTo({ canvas, ctx }) {
|
||||
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||||
},
|
||||
dispose() {
|
||||
if (disposed) return;
|
||||
disposed = true;
|
||||
try {
|
||||
video.pause();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
video.removeAttribute("src");
|
||||
try {
|
||||
video.load();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function waitForLoadedMetadata({
|
||||
video,
|
||||
}: {
|
||||
video: HTMLVideoElement;
|
||||
}): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const cleanup = () => {
|
||||
video.removeEventListener("loadedmetadata", onLoaded);
|
||||
video.removeEventListener("error", onError);
|
||||
};
|
||||
const onLoaded = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = () => {
|
||||
cleanup();
|
||||
const msg = video.error?.message ?? "unknown error";
|
||||
reject(new Error(`HTMLVideoElement load failed: ${msg}`));
|
||||
};
|
||||
video.addEventListener("loadedmetadata", onLoaded);
|
||||
video.addEventListener("error", onError);
|
||||
});
|
||||
}
|
||||
|
||||
function seekVideoElement({
|
||||
video,
|
||||
time,
|
||||
}: {
|
||||
video: HTMLVideoElement;
|
||||
time: number;
|
||||
}): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const cleanup = () => {
|
||||
video.removeEventListener("seeked", onSeeked);
|
||||
video.removeEventListener("error", onError);
|
||||
};
|
||||
const onSeeked = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = () => {
|
||||
cleanup();
|
||||
reject(new Error("HTMLVideoElement seek failed"));
|
||||
};
|
||||
video.addEventListener("seeked", onSeeked);
|
||||
video.addEventListener("error", onError);
|
||||
try {
|
||||
video.currentTime = time;
|
||||
} catch (err) {
|
||||
cleanup();
|
||||
reject(err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export class HTMLVideoElementSink implements FrameSink {
|
||||
private disposed = false;
|
||||
private readonly handle: VideoElementHandle;
|
||||
private readonly fps: number;
|
||||
|
||||
constructor({
|
||||
handle,
|
||||
fps,
|
||||
}: {
|
||||
handle: VideoElementHandle;
|
||||
fps?: number;
|
||||
}) {
|
||||
this.handle = handle;
|
||||
this.fps = clampFallbackFps(fps);
|
||||
}
|
||||
|
||||
static async create({
|
||||
file,
|
||||
fps,
|
||||
}: {
|
||||
file: File;
|
||||
fps?: number;
|
||||
}): Promise<HTMLVideoElementSink> {
|
||||
const handle = await createHTMLVideoElementHandle({ file });
|
||||
return new HTMLVideoElementSink({ handle, fps: clampFallbackFps(fps) });
|
||||
}
|
||||
|
||||
// eslint-disable-next-line opencut/prefer-object-params -- mirrors mediabunny's CanvasSink.canvases(start, end) so VideoCache can use either sink transparently
|
||||
async *canvases(
|
||||
startTimestamp = 0,
|
||||
endTimestamp?: number,
|
||||
): AsyncGenerator<WrappedCanvas, void, unknown> {
|
||||
const step = 1 / this.fps;
|
||||
const duration = this.handle.duration;
|
||||
const end =
|
||||
typeof endTimestamp === "number" && Number.isFinite(endTimestamp)
|
||||
? Math.min(endTimestamp, duration)
|
||||
: duration;
|
||||
let t = Math.max(0, startTimestamp);
|
||||
|
||||
while (t < end && !this.disposed) {
|
||||
try {
|
||||
await this.handle.seek(t);
|
||||
} catch (error) {
|
||||
console.warn("[HTMLVideoElementSink] seek failed:", error);
|
||||
return;
|
||||
}
|
||||
if (this.disposed) return;
|
||||
|
||||
const canvas = createFrameCanvas({
|
||||
width: this.handle.videoWidth,
|
||||
height: this.handle.videoHeight,
|
||||
});
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) {
|
||||
console.warn(
|
||||
"[HTMLVideoElementSink] 2d context unavailable, aborting iterator",
|
||||
);
|
||||
return;
|
||||
}
|
||||
this.handle.drawTo({ canvas, ctx });
|
||||
|
||||
yield {
|
||||
canvas,
|
||||
timestamp: t,
|
||||
duration: step,
|
||||
};
|
||||
t += step;
|
||||
}
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
if (this.disposed) return;
|
||||
this.disposed = true;
|
||||
this.handle.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
function createFrameCanvas({
|
||||
width,
|
||||
height,
|
||||
}: {
|
||||
width: number;
|
||||
height: number;
|
||||
}): HTMLCanvasElement {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = Math.max(1, Math.round(width));
|
||||
canvas.height = Math.max(1, Math.round(height));
|
||||
return canvas;
|
||||
}
|
||||
|
|
@ -5,10 +5,15 @@ import {
|
|||
CanvasSink,
|
||||
type WrappedCanvas,
|
||||
} from "mediabunny";
|
||||
import {
|
||||
type FrameSink,
|
||||
HTMLVideoElementSink,
|
||||
isWebCodecsAvailable,
|
||||
} from "./html-video-element-sink";
|
||||
|
||||
interface VideoSinkData {
|
||||
input: Input;
|
||||
sink: CanvasSink;
|
||||
sink: FrameSink;
|
||||
dispose: () => void;
|
||||
iterator: AsyncGenerator<WrappedCanvas, void, unknown> | null;
|
||||
currentFrame: WrappedCanvas | null;
|
||||
nextFrame: WrappedCanvas | null;
|
||||
|
|
@ -275,7 +280,15 @@ export class VideoCache {
|
|||
|
||||
const canDecode = await videoTrack.canDecode();
|
||||
if (!canDecode) {
|
||||
throw new Error("Video codec not supported for decoding");
|
||||
if (isWebCodecsAvailable()) {
|
||||
throw new Error("Video codec not supported for decoding");
|
||||
}
|
||||
// WebCodecs API absent (e.g. browsers without VideoDecoder support).
|
||||
// Fall back to an HTMLVideoElement-backed sink which uses the
|
||||
// system's native video decoders via the <video> tag.
|
||||
input.dispose();
|
||||
await this.initializeHTMLVideoElementSink({ mediaId, file });
|
||||
return;
|
||||
}
|
||||
|
||||
const sink = new CanvasSink(videoTrack, {
|
||||
|
|
@ -284,8 +297,8 @@ export class VideoCache {
|
|||
});
|
||||
|
||||
this.sinks.set(mediaId, {
|
||||
input,
|
||||
sink,
|
||||
dispose: () => input.dispose(),
|
||||
iterator: null,
|
||||
currentFrame: null,
|
||||
nextFrame: null,
|
||||
|
|
@ -300,6 +313,29 @@ export class VideoCache {
|
|||
}
|
||||
}
|
||||
|
||||
private async initializeHTMLVideoElementSink({
|
||||
mediaId,
|
||||
file,
|
||||
}: {
|
||||
mediaId: string;
|
||||
file: File;
|
||||
}): Promise<void> {
|
||||
const fallbackSink = await HTMLVideoElementSink.create({ file });
|
||||
this.sinks.set(mediaId, {
|
||||
sink: fallbackSink,
|
||||
dispose: () => fallbackSink.dispose(),
|
||||
iterator: null,
|
||||
currentFrame: null,
|
||||
nextFrame: null,
|
||||
lastTime: -1,
|
||||
prefetching: false,
|
||||
prefetchPromise: null,
|
||||
});
|
||||
console.warn(
|
||||
"[VideoCache] WebCodecs unavailable; using HTMLVideoElement fallback for video preview.",
|
||||
);
|
||||
}
|
||||
|
||||
clearVideo({ mediaId }: { mediaId: string }): void {
|
||||
const sinkData = this.sinks.get(mediaId);
|
||||
if (sinkData) {
|
||||
|
|
@ -307,7 +343,7 @@ export class VideoCache {
|
|||
void sinkData.iterator.return();
|
||||
}
|
||||
|
||||
sinkData.input.dispose();
|
||||
sinkData.dispose();
|
||||
this.sinks.delete(mediaId);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue