Merge 7204146467 into 238750c025
This commit is contained in:
Andrés Villagrán
GitHub
commit
3c55eab0a3
|
|
@ -0,0 +1,144 @@
|
|||
import { type NextRequest, NextResponse } from "next/server";
|
||||
import { writeFile, readFile, rm, mkdtemp } from "fs/promises";
|
||||
import { spawn } from "child_process";
|
||||
import path from "path";
|
||||
import os from "os";
|
||||
|
||||
const GENERIC_CONVERSION_ERROR = "Video conversion failed.";
|
||||
|
||||
function sanitizeContentDispositionName({
|
||||
rawName,
|
||||
}: {
|
||||
rawName: string;
|
||||
}): string {
|
||||
// Strip characters that can break the Content-Disposition header (quotes,
|
||||
// backslashes, CR/LF, control bytes) and replace non-ASCII so the
|
||||
// "filename=" parameter stays unambiguous across clients.
|
||||
const stripped = rawName
|
||||
// eslint-disable-next-line no-control-regex
|
||||
.replace(/[\\"\r\n\x00-\x1f]/g, "_")
|
||||
.replace(/[^\x20-\x7e]/g, "_")
|
||||
.slice(0, 200);
|
||||
return stripped.length > 0 ? stripped : "video";
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
let workDir: string | null = null;
|
||||
|
||||
try {
|
||||
const formData = await request.formData();
|
||||
const file = formData.get("file") as File | null;
|
||||
|
||||
if (!file || !(file instanceof File)) {
|
||||
return NextResponse.json(
|
||||
{ error: "No file provided" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
workDir = await mkdtemp(path.join(os.tmpdir(), "opencut-convert-"));
|
||||
const ext = path.extname(file.name) || ".mp4";
|
||||
const inputPath = path.join(workDir, `input${ext}`);
|
||||
const outputPath = path.join(workDir, "output.mp4");
|
||||
|
||||
const buffer = Buffer.from(await file.arrayBuffer());
|
||||
await writeFile(inputPath, buffer);
|
||||
|
||||
await runFfmpeg({
|
||||
inputPath,
|
||||
outputPath,
|
||||
});
|
||||
|
||||
const convertedBuffer = await readFile(outputPath);
|
||||
|
||||
const safeName = sanitizeContentDispositionName({
|
||||
rawName: path.basename(file.name, ext),
|
||||
});
|
||||
|
||||
return new Response(new Uint8Array(convertedBuffer), {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": "video/mp4",
|
||||
"Content-Disposition": `attachment; filename="${safeName}.mp4"`,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Video conversion error:", error);
|
||||
return NextResponse.json(
|
||||
{ error: GENERIC_CONVERSION_ERROR },
|
||||
{ status: 500 },
|
||||
);
|
||||
} finally {
|
||||
if (workDir) {
|
||||
await rm(workDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function runFfmpeg({
|
||||
inputPath,
|
||||
outputPath,
|
||||
}: {
|
||||
inputPath: string;
|
||||
outputPath: string;
|
||||
}): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const proc = spawn("ffmpeg", [
|
||||
"-i",
|
||||
inputPath,
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-profile:v",
|
||||
"baseline",
|
||||
"-level",
|
||||
"3.1",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"23",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-g",
|
||||
"60",
|
||||
"-keyint_min",
|
||||
"60",
|
||||
"-sc_threshold",
|
||||
"0",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
"-tag:v",
|
||||
"avc1",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-ac",
|
||||
"2",
|
||||
"-ar",
|
||||
"48000",
|
||||
"-b:a",
|
||||
"128k",
|
||||
"-y",
|
||||
outputPath,
|
||||
]);
|
||||
|
||||
let stderr = "";
|
||||
proc.stderr.on("data", (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
proc.on("close", (code) => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(
|
||||
new Error(
|
||||
`ffmpeg exited with code ${code}. stderr: ${stderr.slice(-500)}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
proc.on("error", (err) => {
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
@ -20,6 +20,10 @@ import {
|
|||
Input,
|
||||
type WrappedAudioBuffer,
|
||||
} from "mediabunny";
|
||||
import {
|
||||
type AudioBufferSinkLike,
|
||||
FallbackAudioBufferSink,
|
||||
} from "@/services/audio-cache/fallback-audio-buffer-sink";
|
||||
|
||||
export class AudioManager {
|
||||
private audioContext: AudioContext | null = null;
|
||||
|
|
@ -30,7 +34,7 @@ export class AudioManager {
|
|||
private lookaheadSeconds = 2;
|
||||
private scheduleIntervalMs = 500;
|
||||
private clips: AudioClipSource[] = [];
|
||||
private sinks = new Map<string, AudioBufferSink>();
|
||||
private sinks = new Map<string, AudioBufferSinkLike>();
|
||||
private inputs = new Map<string, Input>();
|
||||
private activeClipIds = new Set<string>();
|
||||
private clipIterators = new Map<
|
||||
|
|
@ -267,7 +271,42 @@ export class AudioManager {
|
|||
|
||||
const iterator = sink.buffers(sourceStartTime);
|
||||
this.clipIterators.set(clip.id, iterator);
|
||||
let consecutiveDroppedBufferCount = 0;
|
||||
|
||||
try {
|
||||
await this.consumeClipIterator({
|
||||
clip,
|
||||
clipEnd,
|
||||
iterator,
|
||||
audioContext,
|
||||
sessionId,
|
||||
initialDroppedCount: 0,
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`[AudioManager] clip iterator failed for ${clip.id}; aborting playback for this clip.`,
|
||||
error,
|
||||
);
|
||||
this.clipIterators.delete(clip.id);
|
||||
this.activeClipIds.delete(clip.id);
|
||||
}
|
||||
}
|
||||
|
||||
private async consumeClipIterator({
|
||||
clip,
|
||||
clipEnd,
|
||||
iterator,
|
||||
audioContext,
|
||||
sessionId,
|
||||
initialDroppedCount,
|
||||
}: {
|
||||
clip: AudioClipSource;
|
||||
clipEnd: number;
|
||||
iterator: AsyncGenerator<WrappedAudioBuffer, void, unknown>;
|
||||
audioContext: AudioContext;
|
||||
sessionId: number;
|
||||
initialDroppedCount: number;
|
||||
}): Promise<void> {
|
||||
let consecutiveDroppedBufferCount = initialDroppedCount;
|
||||
|
||||
for await (const { buffer, timestamp } of iterator) {
|
||||
if (!this.editor.playback.getIsPlaying()) return;
|
||||
|
|
@ -605,7 +644,18 @@ export class AudioManager {
|
|||
return null;
|
||||
}
|
||||
|
||||
const sink = new AudioBufferSink(audioTrack);
|
||||
const canDecode = await safeCanDecode({ audioTrack });
|
||||
const sink: AudioBufferSinkLike = canDecode
|
||||
? new AudioBufferSink(audioTrack)
|
||||
: await FallbackAudioBufferSink.create({
|
||||
file: clip.file,
|
||||
audioContext,
|
||||
});
|
||||
if (!canDecode) {
|
||||
console.warn(
|
||||
"[AudioManager] WebCodecs unavailable; decoding clip audio via decodeAudioData.",
|
||||
);
|
||||
}
|
||||
const chunks: AudioBuffer[] = [];
|
||||
let totalSamples = 0;
|
||||
|
||||
|
|
@ -673,12 +723,13 @@ export class AudioManager {
|
|||
clip,
|
||||
}: {
|
||||
clip: AudioClipSource;
|
||||
}): Promise<AudioBufferSink | null> {
|
||||
}): Promise<AudioBufferSinkLike | null> {
|
||||
const existingSink = this.sinks.get(clip.sourceKey);
|
||||
if (existingSink) return existingSink;
|
||||
|
||||
let input: Input | null = null;
|
||||
try {
|
||||
const input = new Input({
|
||||
input = new Input({
|
||||
source: new BlobSource(clip.file),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
|
@ -688,13 +739,47 @@ export class AudioManager {
|
|||
return null;
|
||||
}
|
||||
|
||||
const canDecode = await safeCanDecode({ audioTrack });
|
||||
if (!canDecode) {
|
||||
input.dispose();
|
||||
input = null;
|
||||
const audioContext = this.ensureAudioContext();
|
||||
if (!audioContext) {
|
||||
return null;
|
||||
}
|
||||
const fallback = await FallbackAudioBufferSink.create({
|
||||
file: clip.file,
|
||||
audioContext,
|
||||
});
|
||||
this.sinks.set(clip.sourceKey, fallback);
|
||||
console.warn(
|
||||
"[AudioManager] WebCodecs unavailable; using decodeAudioData fallback for audio clip.",
|
||||
);
|
||||
return fallback;
|
||||
}
|
||||
|
||||
const sink = new AudioBufferSink(audioTrack);
|
||||
this.inputs.set(clip.sourceKey, input);
|
||||
this.sinks.set(clip.sourceKey, sink);
|
||||
return sink;
|
||||
} catch (error) {
|
||||
if (input) {
|
||||
input.dispose();
|
||||
}
|
||||
console.warn("Failed to initialize audio sink:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function safeCanDecode({
|
||||
audioTrack,
|
||||
}: {
|
||||
audioTrack: { canDecode: () => Promise<boolean> };
|
||||
}): Promise<boolean> {
|
||||
try {
|
||||
return await audioTrack.canDecode();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,10 @@ import { canTrackHaveAudio } from "@/timeline";
|
|||
import { mediaSupportsAudio } from "@/media/media-utils";
|
||||
import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/retime";
|
||||
import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny";
|
||||
import {
|
||||
type AudioBufferSinkLike,
|
||||
FallbackAudioBufferSink,
|
||||
} from "@/services/audio-cache/fallback-audio-buffer-sink";
|
||||
import { TICKS_PER_SECOND } from "@/wasm";
|
||||
import {
|
||||
computeRmsBuckets,
|
||||
|
|
@ -271,7 +275,23 @@ async function resolveAudioBufferForAsset({
|
|||
const audioTrack = await input.getPrimaryAudioTrack();
|
||||
if (!audioTrack) return null;
|
||||
|
||||
const sink = new AudioBufferSink(audioTrack);
|
||||
let audioTrackCanDecode = false;
|
||||
try {
|
||||
audioTrackCanDecode = await audioTrack.canDecode();
|
||||
} catch {
|
||||
audioTrackCanDecode = false;
|
||||
}
|
||||
const sink: AudioBufferSinkLike = audioTrackCanDecode
|
||||
? new AudioBufferSink(audioTrack)
|
||||
: await FallbackAudioBufferSink.create({
|
||||
file: asset.file,
|
||||
audioContext,
|
||||
});
|
||||
if (!audioTrackCanDecode) {
|
||||
console.warn(
|
||||
"[audio] WebCodecs unavailable; decoding asset via decodeAudioData fallback.",
|
||||
);
|
||||
}
|
||||
const targetSampleRate = audioContext.sampleRate;
|
||||
|
||||
const chunks: AudioBuffer[] = [];
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
export async function convertVideoToH264({
|
||||
file,
|
||||
}: {
|
||||
file: File;
|
||||
}): Promise<File> {
|
||||
const formData = new FormData();
|
||||
formData.append("file", file);
|
||||
|
||||
const response = await fetch("/api/convert-video", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}));
|
||||
throw new Error(errorData.error || `Conversion failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
const baseName = file.name.replace(/\.[^.]+$/, "");
|
||||
return new File([blob], `${baseName}.mp4`, { type: "video/mp4" });
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ import type { MediaAsset } from "@/media/types";
|
|||
import { readVideoFile } from "./mediabunny";
|
||||
import type { VideoFileData } from "./mediabunny";
|
||||
import { renderThumbnailDataUrl } from "./thumbnail";
|
||||
import { convertVideoToH264 } from "./ffmpeg-convert";
|
||||
|
||||
export interface ProcessedMediaAsset extends Omit<MediaAsset, "id"> {}
|
||||
|
||||
|
|
@ -132,8 +133,49 @@ export async function processMediaAssets({
|
|||
width = result.width;
|
||||
height = result.height;
|
||||
} else if (fileType === "video") {
|
||||
let currentFile = file;
|
||||
let currentUrl = url;
|
||||
try {
|
||||
const videoData = await readVideoFile({ file });
|
||||
let videoData = await readVideoFile({ file: currentFile });
|
||||
|
||||
if (!videoData.canDecode) {
|
||||
toast.info(`Converting ${file.name} to a compatible format…`);
|
||||
const convertedFile = await convertVideoToH264({
|
||||
file: currentFile,
|
||||
});
|
||||
|
||||
const convertedStorageCheck = await storageService.canStoreFile({
|
||||
size: convertedFile.size,
|
||||
});
|
||||
if (!convertedStorageCheck.canStore) {
|
||||
throw new Error(
|
||||
getStorageLimitDescription({
|
||||
fileSize: convertedFile.size,
|
||||
availableBytes: convertedStorageCheck.availableBytes,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
URL.revokeObjectURL(currentUrl);
|
||||
currentFile = convertedFile;
|
||||
currentUrl = URL.createObjectURL(currentFile);
|
||||
videoData = await readVideoFile({ file: currentFile });
|
||||
|
||||
if (!videoData.canDecode) {
|
||||
// Conversion produced a file the browser still can't decode
|
||||
// via mediabunny/WebCodecs. The HTMLVideoElement fallback in
|
||||
// VideoCache will handle preview rendering, so warn but
|
||||
// continue importing.
|
||||
toast.warning(`${file.name} converted, preview may be limited`, {
|
||||
description: getUnsupportedVideoDescription({
|
||||
codec: videoData.codec,
|
||||
}),
|
||||
});
|
||||
} else {
|
||||
toast.success(`${file.name} converted successfully`);
|
||||
}
|
||||
}
|
||||
|
||||
duration = videoData.duration;
|
||||
width = videoData.width;
|
||||
height = videoData.height;
|
||||
|
|
@ -143,13 +185,27 @@ export async function processMediaAssets({
|
|||
hasAudio = videoData.hasAudio;
|
||||
thumbnailUrl = videoData.thumbnailUrl ?? undefined;
|
||||
|
||||
if (!videoData.canDecode) {
|
||||
toast.error(`Can't preview ${file.name}`, {
|
||||
description: getUnsupportedVideoDescription({
|
||||
codec: videoData.codec,
|
||||
}),
|
||||
});
|
||||
processedAssets.push({
|
||||
name: currentFile.name,
|
||||
type: fileType,
|
||||
file: currentFile,
|
||||
url: currentUrl,
|
||||
thumbnailUrl,
|
||||
duration,
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
hasAudio,
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
completed += 1;
|
||||
if (onProgress) {
|
||||
const percent = Math.round((completed / total) * 100);
|
||||
onProgress({ progress: percent });
|
||||
}
|
||||
continue;
|
||||
} catch (error) {
|
||||
const message =
|
||||
error instanceof Error
|
||||
|
|
@ -159,6 +215,8 @@ export async function processMediaAssets({
|
|||
toast.error(`Couldn't process ${file.name}`, {
|
||||
description: message,
|
||||
});
|
||||
URL.revokeObjectURL(currentUrl);
|
||||
continue;
|
||||
}
|
||||
} else if (fileType === "audio") {
|
||||
duration = await getMediaDuration({ file });
|
||||
|
|
|
|||
|
|
@ -0,0 +1,308 @@
|
|||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
test,
|
||||
} from "bun:test";
|
||||
import {
|
||||
FallbackAudioBufferSink,
|
||||
isWebAudioDecoderAvailable,
|
||||
sliceAudioBuffer,
|
||||
} from "../fallback-audio-buffer-sink";
|
||||
|
||||
type AudioBufferLike = {
|
||||
length: number;
|
||||
sampleRate: number;
|
||||
numberOfChannels: number;
|
||||
duration: number;
|
||||
getChannelData: (channel: number) => Float32Array;
|
||||
copyToChannel: (
|
||||
source: Float32Array,
|
||||
channel: number,
|
||||
startInChannel?: number,
|
||||
) => void;
|
||||
};
|
||||
|
||||
function createTestAudioBuffer({
|
||||
durationSeconds = 1,
|
||||
sampleRate = 8,
|
||||
channelCount = 2,
|
||||
}: {
|
||||
durationSeconds?: number;
|
||||
sampleRate?: number;
|
||||
channelCount?: number;
|
||||
} = {}): AudioBufferLike {
|
||||
const length = Math.round(durationSeconds * sampleRate);
|
||||
const channels = Array.from(
|
||||
{ length: channelCount },
|
||||
(_, channelIndex) =>
|
||||
new Float32Array(
|
||||
Array.from({ length }, (_, i) => channelIndex + 1 + i * 0.01),
|
||||
),
|
||||
);
|
||||
return {
|
||||
length,
|
||||
sampleRate,
|
||||
numberOfChannels: channelCount,
|
||||
duration: durationSeconds,
|
||||
getChannelData(channel: number) {
|
||||
return channels[channel];
|
||||
},
|
||||
// eslint-disable-next-line opencut/prefer-object-params -- mirrors the AudioBuffer.copyToChannel Web API signature so this fake behaves like the real one
|
||||
copyToChannel(
|
||||
source: Float32Array,
|
||||
channel: number,
|
||||
startInChannel = 0,
|
||||
) {
|
||||
channels[channel].set(source, startInChannel);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function installFakeAudioBuffer(): { restore: () => void } {
|
||||
const root = globalThis as { AudioBuffer?: unknown };
|
||||
const original = root.AudioBuffer;
|
||||
class FakeAudioBuffer implements AudioBufferLike {
|
||||
length: number;
|
||||
sampleRate: number;
|
||||
numberOfChannels: number;
|
||||
duration: number;
|
||||
private channels: Float32Array[];
|
||||
|
||||
constructor(opts: {
|
||||
length: number;
|
||||
numberOfChannels: number;
|
||||
sampleRate: number;
|
||||
}) {
|
||||
this.length = opts.length;
|
||||
this.sampleRate = opts.sampleRate;
|
||||
this.numberOfChannels = opts.numberOfChannels;
|
||||
this.duration = opts.length / opts.sampleRate;
|
||||
this.channels = Array.from(
|
||||
{ length: opts.numberOfChannels },
|
||||
() => new Float32Array(opts.length),
|
||||
);
|
||||
}
|
||||
|
||||
getChannelData(channel: number): Float32Array {
|
||||
return this.channels[channel];
|
||||
}
|
||||
|
||||
// eslint-disable-next-line opencut/prefer-object-params -- mirrors AudioBuffer.copyToChannel Web API signature
|
||||
copyToChannel(
|
||||
source: Float32Array,
|
||||
channel: number,
|
||||
startInChannel = 0,
|
||||
): void {
|
||||
this.channels[channel].set(source, startInChannel);
|
||||
}
|
||||
}
|
||||
root.AudioBuffer = FakeAudioBuffer;
|
||||
return {
|
||||
restore() {
|
||||
if (original === undefined) {
|
||||
delete root.AudioBuffer;
|
||||
} else {
|
||||
root.AudioBuffer = original;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("isWebAudioDecoderAvailable", () => {
|
||||
const root = globalThis as { AudioDecoder?: unknown };
|
||||
let original: unknown;
|
||||
|
||||
beforeEach(() => {
|
||||
original = root.AudioDecoder;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (original === undefined) {
|
||||
delete root.AudioDecoder;
|
||||
} else {
|
||||
root.AudioDecoder = original;
|
||||
}
|
||||
});
|
||||
|
||||
test("returns false when AudioDecoder undefined", () => {
|
||||
delete root.AudioDecoder;
|
||||
expect(isWebAudioDecoderAvailable()).toBe(false);
|
||||
});
|
||||
|
||||
test("returns true when AudioDecoder defined", () => {
|
||||
root.AudioDecoder = class {};
|
||||
expect(isWebAudioDecoderAvailable()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sliceAudioBuffer", () => {
|
||||
let fakeAudioBuffer: ReturnType<typeof installFakeAudioBuffer>;
|
||||
|
||||
beforeEach(() => {
|
||||
fakeAudioBuffer = installFakeAudioBuffer();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fakeAudioBuffer.restore();
|
||||
});
|
||||
|
||||
test("slices to the requested range", () => {
|
||||
const buffer = createTestAudioBuffer({
|
||||
durationSeconds: 1,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sliced = sliceAudioBuffer({
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
buffer: buffer as unknown as AudioBuffer,
|
||||
start: 0.25,
|
||||
end: 0.75,
|
||||
});
|
||||
expect(sliced.length).toBe(4);
|
||||
expect(sliced.numberOfChannels).toBe(1);
|
||||
expect(sliced.sampleRate).toBe(8);
|
||||
const channel = sliced.getChannelData(0);
|
||||
expect(channel[0]).toBeCloseTo(1 + 0.02, 5);
|
||||
expect(channel[3]).toBeCloseTo(1 + 0.05, 5);
|
||||
});
|
||||
|
||||
test("clamps out-of-range start/end", () => {
|
||||
const buffer = createTestAudioBuffer({
|
||||
durationSeconds: 0.5,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sliced = sliceAudioBuffer({
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
buffer: buffer as unknown as AudioBuffer,
|
||||
start: -1,
|
||||
end: 5,
|
||||
});
|
||||
expect(sliced.length).toBe(buffer.length);
|
||||
});
|
||||
|
||||
test("preserves channel count", () => {
|
||||
const buffer = createTestAudioBuffer({
|
||||
durationSeconds: 1,
|
||||
sampleRate: 8,
|
||||
channelCount: 2,
|
||||
});
|
||||
const sliced = sliceAudioBuffer({
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
buffer: buffer as unknown as AudioBuffer,
|
||||
start: 0,
|
||||
end: 0.5,
|
||||
});
|
||||
expect(sliced.numberOfChannels).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("FallbackAudioBufferSink.buffers", () => {
|
||||
let fakeAudioBuffer: ReturnType<typeof installFakeAudioBuffer>;
|
||||
|
||||
beforeEach(() => {
|
||||
fakeAudioBuffer = installFakeAudioBuffer();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fakeAudioBuffer.restore();
|
||||
});
|
||||
|
||||
test("yields a single buffer covering the full range by default", async () => {
|
||||
const decoded = createTestAudioBuffer({
|
||||
durationSeconds: 1,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sink = new FallbackAudioBufferSink(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
decoded as unknown as AudioBuffer,
|
||||
);
|
||||
const yielded = [];
|
||||
for await (const wrapped of sink.buffers()) {
|
||||
yielded.push(wrapped);
|
||||
}
|
||||
expect(yielded).toHaveLength(1);
|
||||
expect(yielded[0].timestamp).toBe(0);
|
||||
expect(yielded[0].duration).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
test("yields a slice when startTimestamp/endTimestamp provided", async () => {
|
||||
const decoded = createTestAudioBuffer({
|
||||
durationSeconds: 2,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sink = new FallbackAudioBufferSink(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
decoded as unknown as AudioBuffer,
|
||||
);
|
||||
const yielded = [];
|
||||
for await (const wrapped of sink.buffers(0.5, 1.5)) {
|
||||
yielded.push(wrapped);
|
||||
}
|
||||
expect(yielded).toHaveLength(1);
|
||||
expect(yielded[0].timestamp).toBe(0.5);
|
||||
expect(yielded[0].duration).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
test("treats NaN/Infinity startTimestamp as 0", async () => {
|
||||
const decoded = createTestAudioBuffer({
|
||||
durationSeconds: 1,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sink = new FallbackAudioBufferSink(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
decoded as unknown as AudioBuffer,
|
||||
);
|
||||
const yielded = [];
|
||||
for await (const wrapped of sink.buffers(Number.NaN)) {
|
||||
yielded.push(wrapped);
|
||||
}
|
||||
expect(yielded).toHaveLength(1);
|
||||
expect(yielded[0].timestamp).toBe(0);
|
||||
expect(yielded[0].duration).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
test("yields nothing when start >= end (or beyond duration)", async () => {
|
||||
const decoded = createTestAudioBuffer({
|
||||
durationSeconds: 1,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sink = new FallbackAudioBufferSink(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
decoded as unknown as AudioBuffer,
|
||||
);
|
||||
|
||||
const yielded = [];
|
||||
for await (const wrapped of sink.buffers(2)) {
|
||||
yielded.push(wrapped);
|
||||
}
|
||||
expect(yielded).toHaveLength(0);
|
||||
|
||||
const yielded2 = [];
|
||||
for await (const wrapped of sink.buffers(0.4, 0.4)) {
|
||||
yielded2.push(wrapped);
|
||||
}
|
||||
expect(yielded2).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("exposes sampleRate/numberOfChannels/duration getters", () => {
|
||||
const decoded = createTestAudioBuffer({
|
||||
durationSeconds: 3,
|
||||
sampleRate: 16,
|
||||
channelCount: 2,
|
||||
});
|
||||
const sink = new FallbackAudioBufferSink(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
decoded as unknown as AudioBuffer,
|
||||
);
|
||||
expect(sink.sampleRate).toBe(16);
|
||||
expect(sink.numberOfChannels).toBe(2);
|
||||
expect(sink.duration).toBe(3);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
import type { WrappedAudioBuffer } from "mediabunny";
|
||||
|
||||
export interface AudioBufferSinkLike {
|
||||
buffers(
|
||||
startTimestamp?: number,
|
||||
endTimestamp?: number,
|
||||
): AsyncGenerator<WrappedAudioBuffer, void, unknown>;
|
||||
}
|
||||
|
||||
export function isWebAudioDecoderAvailable(): boolean {
|
||||
return (
|
||||
typeof (globalThis as { AudioDecoder?: unknown }).AudioDecoder !==
|
||||
"undefined"
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* AudioBufferSink implementation backed by AudioContext.decodeAudioData.
|
||||
*
|
||||
* Used when mediabunny's AudioSampleSink cannot decode the track because the
|
||||
* environment lacks the WebCodecs AudioDecoder API. AudioContext.decodeAudioData
|
||||
* uses the system's native audio decoders, mirroring the strategy used by
|
||||
* HTMLVideoElementSink for video. The entire compressed audio is decoded
|
||||
* upfront and exposed through a buffers() iterator that yields a single
|
||||
* AudioBuffer slice covering the requested range.
|
||||
*/
|
||||
export class FallbackAudioBufferSink implements AudioBufferSinkLike {
|
||||
constructor(private readonly decoded: AudioBuffer) {}
|
||||
|
||||
static async create({
|
||||
file,
|
||||
audioContext,
|
||||
}: {
|
||||
file: File;
|
||||
audioContext: BaseAudioContext;
|
||||
}): Promise<FallbackAudioBufferSink> {
|
||||
const arrayBuffer = await file.arrayBuffer();
|
||||
const decoded = await decodeArrayBuffer({ arrayBuffer, audioContext });
|
||||
return new FallbackAudioBufferSink(decoded);
|
||||
}
|
||||
|
||||
get sampleRate(): number {
|
||||
return this.decoded.sampleRate;
|
||||
}
|
||||
|
||||
get numberOfChannels(): number {
|
||||
return this.decoded.numberOfChannels;
|
||||
}
|
||||
|
||||
get duration(): number {
|
||||
return this.decoded.duration;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line opencut/prefer-object-params -- mirrors mediabunny's AudioBufferSink.buffers(start, end) so consumers can swap sinks transparently
|
||||
async *buffers(
|
||||
startTimestamp = 0,
|
||||
endTimestamp?: number,
|
||||
): AsyncGenerator<WrappedAudioBuffer, void, unknown> {
|
||||
const totalDuration = this.decoded.duration;
|
||||
const safeStart = Number.isFinite(startTimestamp) ? startTimestamp : 0;
|
||||
const start = Math.max(0, Math.min(safeStart, totalDuration));
|
||||
const end =
|
||||
typeof endTimestamp === "number" && Number.isFinite(endTimestamp)
|
||||
? Math.min(endTimestamp, totalDuration)
|
||||
: totalDuration;
|
||||
if (start >= end) return;
|
||||
|
||||
const sliced = sliceAudioBuffer({
|
||||
buffer: this.decoded,
|
||||
start,
|
||||
end,
|
||||
});
|
||||
|
||||
yield {
|
||||
buffer: sliced,
|
||||
timestamp: start,
|
||||
duration: sliced.duration,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function decodeArrayBuffer({
|
||||
arrayBuffer,
|
||||
audioContext,
|
||||
}: {
|
||||
arrayBuffer: ArrayBuffer;
|
||||
audioContext: BaseAudioContext;
|
||||
}): Promise<AudioBuffer> {
|
||||
return new Promise<AudioBuffer>((resolve, reject) => {
|
||||
audioContext.decodeAudioData(
|
||||
arrayBuffer,
|
||||
(buffer) => resolve(buffer),
|
||||
(err) =>
|
||||
reject(
|
||||
err instanceof Error
|
||||
? err
|
||||
: new Error(`decodeAudioData failed: ${String(err)}`),
|
||||
),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export function sliceAudioBuffer({
|
||||
buffer,
|
||||
start,
|
||||
end,
|
||||
}: {
|
||||
buffer: AudioBuffer;
|
||||
start: number;
|
||||
end: number;
|
||||
}): AudioBuffer {
|
||||
const sampleRate = buffer.sampleRate;
|
||||
const startSample = Math.max(
|
||||
0,
|
||||
Math.min(buffer.length, Math.floor(start * sampleRate)),
|
||||
);
|
||||
const endSample = Math.max(
|
||||
startSample,
|
||||
Math.min(buffer.length, Math.ceil(end * sampleRate)),
|
||||
);
|
||||
const length = Math.max(1, endSample - startSample);
|
||||
|
||||
const target = new AudioBuffer({
|
||||
length,
|
||||
numberOfChannels: Math.max(1, buffer.numberOfChannels),
|
||||
sampleRate,
|
||||
});
|
||||
for (let channel = 0; channel < buffer.numberOfChannels; channel++) {
|
||||
const channelView = buffer
|
||||
.getChannelData(channel)
|
||||
.subarray(startSample, endSample);
|
||||
target.copyToChannel(channelView, channel, 0);
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
|
@ -0,0 +1,259 @@
|
|||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
test,
|
||||
} from "bun:test";
|
||||
import {
|
||||
clampFallbackFps,
|
||||
HTMLVideoElementSink,
|
||||
isWebCodecsAvailable,
|
||||
type VideoElementHandle,
|
||||
} from "../html-video-element-sink";
|
||||
|
||||
type FakeCanvas = {
|
||||
width: number;
|
||||
height: number;
|
||||
getContext: (kind: string) => FakeCanvasContext | null;
|
||||
};
|
||||
|
||||
type FakeCanvasContext = {
|
||||
drawImage: (...args: unknown[]) => void;
|
||||
drawnCount: number;
|
||||
};
|
||||
|
||||
function installFakeDom(): {
|
||||
createdCanvases: FakeCanvas[];
|
||||
restore: () => void;
|
||||
} {
|
||||
const createdCanvases: FakeCanvas[] = [];
|
||||
const originalDocument = (
|
||||
globalThis as { document?: unknown }
|
||||
).document;
|
||||
|
||||
const fakeDocument = {
|
||||
createElement(tag: string) {
|
||||
if (tag !== "canvas") {
|
||||
throw new Error(`unexpected createElement(${tag}) in test`);
|
||||
}
|
||||
const ctx: FakeCanvasContext = {
|
||||
drawnCount: 0,
|
||||
drawImage() {
|
||||
ctx.drawnCount += 1;
|
||||
},
|
||||
};
|
||||
const canvas: FakeCanvas = {
|
||||
width: 0,
|
||||
height: 0,
|
||||
getContext(kind: string) {
|
||||
return kind === "2d" ? ctx : null;
|
||||
},
|
||||
};
|
||||
createdCanvases.push(canvas);
|
||||
return canvas;
|
||||
},
|
||||
};
|
||||
|
||||
(globalThis as { document?: unknown }).document = fakeDocument;
|
||||
|
||||
return {
|
||||
createdCanvases,
|
||||
restore() {
|
||||
(globalThis as { document?: unknown }).document = originalDocument;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createFakeHandle({
|
||||
duration = 2,
|
||||
videoWidth = 640,
|
||||
videoHeight = 360,
|
||||
}: {
|
||||
duration?: number;
|
||||
videoWidth?: number;
|
||||
videoHeight?: number;
|
||||
} = {}): VideoElementHandle & {
|
||||
seekCalls: number[];
|
||||
drawCalls: number;
|
||||
disposed: boolean;
|
||||
} {
|
||||
const handle = {
|
||||
seekCalls: [] as number[],
|
||||
drawCalls: 0,
|
||||
disposed: false,
|
||||
duration,
|
||||
videoWidth,
|
||||
videoHeight,
|
||||
async seek(time: number) {
|
||||
handle.seekCalls.push(time);
|
||||
},
|
||||
drawTo() {
|
||||
handle.drawCalls += 1;
|
||||
},
|
||||
dispose() {
|
||||
handle.disposed = true;
|
||||
},
|
||||
};
|
||||
return handle;
|
||||
}
|
||||
|
||||
describe("clampFallbackFps", () => {
|
||||
test("returns default for undefined/NaN/zero/negative", () => {
|
||||
expect(clampFallbackFps(undefined)).toBe(30);
|
||||
expect(clampFallbackFps(Number.NaN)).toBe(30);
|
||||
expect(clampFallbackFps(0)).toBe(30);
|
||||
expect(clampFallbackFps(-5)).toBe(30);
|
||||
});
|
||||
|
||||
test("rounds and clamps within [1, 60]", () => {
|
||||
expect(clampFallbackFps(24)).toBe(24);
|
||||
expect(clampFallbackFps(29.4)).toBe(29);
|
||||
expect(clampFallbackFps(120)).toBe(60);
|
||||
expect(clampFallbackFps(0.4)).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isWebCodecsAvailable", () => {
|
||||
const root = globalThis as { VideoDecoder?: unknown };
|
||||
let originalVideoDecoder: unknown;
|
||||
|
||||
beforeEach(() => {
|
||||
originalVideoDecoder = root.VideoDecoder;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (originalVideoDecoder === undefined) {
|
||||
delete root.VideoDecoder;
|
||||
} else {
|
||||
root.VideoDecoder = originalVideoDecoder;
|
||||
}
|
||||
});
|
||||
|
||||
test("false when VideoDecoder is undefined", () => {
|
||||
delete root.VideoDecoder;
|
||||
expect(isWebCodecsAvailable()).toBe(false);
|
||||
});
|
||||
|
||||
test("true when VideoDecoder is defined", () => {
|
||||
root.VideoDecoder = class {};
|
||||
expect(isWebCodecsAvailable()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("HTMLVideoElementSink.canvases", () => {
|
||||
let dom: ReturnType<typeof installFakeDom>;
|
||||
|
||||
beforeEach(() => {
|
||||
dom = installFakeDom();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
dom.restore();
|
||||
});
|
||||
|
||||
test("yields frames stepping by 1/fps from startTimestamp", async () => {
|
||||
const handle = createFakeHandle({ duration: 0.25 });
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 10 }); // step = 0.1s
|
||||
|
||||
const frames: Array<{ timestamp: number; duration: number }> = [];
|
||||
for await (const frame of sink.canvases(0)) {
|
||||
frames.push({
|
||||
timestamp: frame.timestamp,
|
||||
duration: frame.duration,
|
||||
});
|
||||
}
|
||||
|
||||
expect(frames).toHaveLength(3);
|
||||
expect(frames[0].timestamp).toBeCloseTo(0, 5);
|
||||
expect(frames[1].timestamp).toBeCloseTo(0.1, 5);
|
||||
expect(frames[2].timestamp).toBeCloseTo(0.2, 5);
|
||||
expect(frames[0].duration).toBeCloseTo(0.1, 5);
|
||||
expect(handle.seekCalls).toEqual([0, 0.1, 0.2]);
|
||||
expect(handle.drawCalls).toBe(3);
|
||||
});
|
||||
|
||||
test("respects endTimestamp", async () => {
|
||||
const handle = createFakeHandle({ duration: 5 });
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 10 });
|
||||
|
||||
const frames: number[] = [];
|
||||
for await (const frame of sink.canvases(0, 0.2)) {
|
||||
frames.push(frame.timestamp);
|
||||
}
|
||||
|
||||
expect(frames).toHaveLength(2);
|
||||
expect(frames[0]).toBeCloseTo(0, 5);
|
||||
expect(frames[1]).toBeCloseTo(0.1, 5);
|
||||
});
|
||||
|
||||
test("dispose stops the iterator early", async () => {
|
||||
const handle = createFakeHandle({ duration: 10 });
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 10 });
|
||||
|
||||
const collected: number[] = [];
|
||||
for await (const frame of sink.canvases(0)) {
|
||||
collected.push(frame.timestamp);
|
||||
if (collected.length === 2) sink.dispose();
|
||||
}
|
||||
|
||||
expect(collected).toHaveLength(2);
|
||||
expect(handle.disposed).toBe(true);
|
||||
});
|
||||
|
||||
test("returns no frames when start >= duration", async () => {
|
||||
const handle = createFakeHandle({ duration: 1 });
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 30 });
|
||||
|
||||
const frames: number[] = [];
|
||||
for await (const frame of sink.canvases(2)) {
|
||||
frames.push(frame.timestamp);
|
||||
}
|
||||
expect(frames).toHaveLength(0);
|
||||
expect(handle.seekCalls).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("aborts cleanly when seek throws", async () => {
|
||||
const handle = createFakeHandle({ duration: 1 });
|
||||
handle.seek = async () => {
|
||||
throw new Error("boom");
|
||||
};
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 30 });
|
||||
|
||||
const frames: number[] = [];
|
||||
for await (const frame of sink.canvases(0)) {
|
||||
frames.push(frame.timestamp);
|
||||
}
|
||||
expect(frames).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("clamps invalid fps passed to constructor to the default", async () => {
|
||||
const handle = createFakeHandle({ duration: 0.05 });
|
||||
// Passing 0 (negative, or NaN) used to leak through to 1/fps and produce
|
||||
// invalid frame stepping. The constructor must reject these and fall back
|
||||
// to the default FPS.
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 0 });
|
||||
const frames = [];
|
||||
for await (const frame of sink.canvases(0)) {
|
||||
frames.push(frame);
|
||||
}
|
||||
expect(frames).toHaveLength(2);
|
||||
expect(frames[0].duration).toBeCloseTo(1 / 30, 5);
|
||||
});
|
||||
|
||||
test("frame canvas matches video dimensions", async () => {
|
||||
const handle = createFakeHandle({
|
||||
duration: 0.02,
|
||||
videoWidth: 1280,
|
||||
videoHeight: 720,
|
||||
});
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 30 });
|
||||
const frames = [];
|
||||
for await (const frame of sink.canvases(0)) {
|
||||
frames.push(frame);
|
||||
}
|
||||
expect(frames).toHaveLength(1);
|
||||
expect(frames[0].canvas.width).toBe(1280);
|
||||
expect(frames[0].canvas.height).toBe(720);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,249 @@
|
|||
import type { WrappedCanvas } from "mediabunny";
|
||||
|
||||
const DEFAULT_FALLBACK_FPS = 30;
|
||||
const MAX_FALLBACK_FPS = 60;
|
||||
const MIN_FALLBACK_FPS = 1;
|
||||
|
||||
export interface FrameSink {
|
||||
canvases(
|
||||
startTimestamp?: number,
|
||||
endTimestamp?: number,
|
||||
): AsyncGenerator<WrappedCanvas, void, unknown>;
|
||||
}
|
||||
|
||||
export interface VideoElementHandle {
|
||||
readonly duration: number;
|
||||
readonly videoWidth: number;
|
||||
readonly videoHeight: number;
|
||||
seek(time: number): Promise<void>;
|
||||
drawTo(target: {
|
||||
canvas: HTMLCanvasElement | OffscreenCanvas;
|
||||
ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
|
||||
}): void;
|
||||
dispose(): void;
|
||||
}
|
||||
|
||||
export function isWebCodecsAvailable(): boolean {
|
||||
return typeof (globalThis as { VideoDecoder?: unknown }).VideoDecoder !==
|
||||
"undefined";
|
||||
}
|
||||
|
||||
export function clampFallbackFps(requested: number | undefined): number {
|
||||
if (
|
||||
requested === undefined ||
|
||||
!Number.isFinite(requested) ||
|
||||
requested <= 0
|
||||
) {
|
||||
return DEFAULT_FALLBACK_FPS;
|
||||
}
|
||||
const rounded = Math.round(requested);
|
||||
if (rounded < MIN_FALLBACK_FPS) return MIN_FALLBACK_FPS;
|
||||
if (rounded > MAX_FALLBACK_FPS) return MAX_FALLBACK_FPS;
|
||||
return rounded;
|
||||
}
|
||||
|
||||
export async function createHTMLVideoElementHandle({
|
||||
file,
|
||||
}: {
|
||||
file: File;
|
||||
}): Promise<VideoElementHandle> {
|
||||
const objectUrl = URL.createObjectURL(file);
|
||||
const video = document.createElement("video");
|
||||
video.preload = "auto";
|
||||
video.muted = true;
|
||||
video.playsInline = true;
|
||||
video.crossOrigin = "anonymous";
|
||||
video.src = objectUrl;
|
||||
|
||||
try {
|
||||
await waitForLoadedMetadata({ video });
|
||||
} catch (error) {
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
throw error;
|
||||
}
|
||||
|
||||
let disposed = false;
|
||||
|
||||
return {
|
||||
get duration() {
|
||||
return video.duration;
|
||||
},
|
||||
get videoWidth() {
|
||||
return video.videoWidth;
|
||||
},
|
||||
get videoHeight() {
|
||||
return video.videoHeight;
|
||||
},
|
||||
async seek(time) {
|
||||
if (disposed) {
|
||||
throw new Error("VideoElementHandle disposed");
|
||||
}
|
||||
await seekVideoElement({ video, time });
|
||||
},
|
||||
drawTo({ canvas, ctx }) {
|
||||
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||||
},
|
||||
dispose() {
|
||||
if (disposed) return;
|
||||
disposed = true;
|
||||
try {
|
||||
video.pause();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
video.removeAttribute("src");
|
||||
try {
|
||||
video.load();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function waitForLoadedMetadata({
|
||||
video,
|
||||
}: {
|
||||
video: HTMLVideoElement;
|
||||
}): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const cleanup = () => {
|
||||
video.removeEventListener("loadedmetadata", onLoaded);
|
||||
video.removeEventListener("error", onError);
|
||||
};
|
||||
const onLoaded = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = () => {
|
||||
cleanup();
|
||||
const msg = video.error?.message ?? "unknown error";
|
||||
reject(new Error(`HTMLVideoElement load failed: ${msg}`));
|
||||
};
|
||||
video.addEventListener("loadedmetadata", onLoaded);
|
||||
video.addEventListener("error", onError);
|
||||
});
|
||||
}
|
||||
|
||||
function seekVideoElement({
|
||||
video,
|
||||
time,
|
||||
}: {
|
||||
video: HTMLVideoElement;
|
||||
time: number;
|
||||
}): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const cleanup = () => {
|
||||
video.removeEventListener("seeked", onSeeked);
|
||||
video.removeEventListener("error", onError);
|
||||
};
|
||||
const onSeeked = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = () => {
|
||||
cleanup();
|
||||
reject(new Error("HTMLVideoElement seek failed"));
|
||||
};
|
||||
video.addEventListener("seeked", onSeeked);
|
||||
video.addEventListener("error", onError);
|
||||
try {
|
||||
video.currentTime = time;
|
||||
} catch (err) {
|
||||
cleanup();
|
||||
reject(err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export class HTMLVideoElementSink implements FrameSink {
|
||||
private disposed = false;
|
||||
private readonly handle: VideoElementHandle;
|
||||
private readonly fps: number;
|
||||
|
||||
constructor({
|
||||
handle,
|
||||
fps,
|
||||
}: {
|
||||
handle: VideoElementHandle;
|
||||
fps?: number;
|
||||
}) {
|
||||
this.handle = handle;
|
||||
this.fps = clampFallbackFps(fps);
|
||||
}
|
||||
|
||||
static async create({
|
||||
file,
|
||||
fps,
|
||||
}: {
|
||||
file: File;
|
||||
fps?: number;
|
||||
}): Promise<HTMLVideoElementSink> {
|
||||
const handle = await createHTMLVideoElementHandle({ file });
|
||||
return new HTMLVideoElementSink({ handle, fps: clampFallbackFps(fps) });
|
||||
}
|
||||
|
||||
// eslint-disable-next-line opencut/prefer-object-params -- mirrors mediabunny's CanvasSink.canvases(start, end) so VideoCache can use either sink transparently
|
||||
async *canvases(
|
||||
startTimestamp = 0,
|
||||
endTimestamp?: number,
|
||||
): AsyncGenerator<WrappedCanvas, void, unknown> {
|
||||
const step = 1 / this.fps;
|
||||
const duration = this.handle.duration;
|
||||
const end =
|
||||
typeof endTimestamp === "number" && Number.isFinite(endTimestamp)
|
||||
? Math.min(endTimestamp, duration)
|
||||
: duration;
|
||||
let t = Math.max(0, startTimestamp);
|
||||
|
||||
while (t < end && !this.disposed) {
|
||||
try {
|
||||
await this.handle.seek(t);
|
||||
} catch (error) {
|
||||
console.warn("[HTMLVideoElementSink] seek failed:", error);
|
||||
return;
|
||||
}
|
||||
if (this.disposed) return;
|
||||
|
||||
const canvas = createFrameCanvas({
|
||||
width: this.handle.videoWidth,
|
||||
height: this.handle.videoHeight,
|
||||
});
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) {
|
||||
console.warn(
|
||||
"[HTMLVideoElementSink] 2d context unavailable, aborting iterator",
|
||||
);
|
||||
return;
|
||||
}
|
||||
this.handle.drawTo({ canvas, ctx });
|
||||
|
||||
yield {
|
||||
canvas,
|
||||
timestamp: t,
|
||||
duration: step,
|
||||
};
|
||||
t += step;
|
||||
}
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
if (this.disposed) return;
|
||||
this.disposed = true;
|
||||
this.handle.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
function createFrameCanvas({
|
||||
width,
|
||||
height,
|
||||
}: {
|
||||
width: number;
|
||||
height: number;
|
||||
}): HTMLCanvasElement {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = Math.max(1, Math.round(width));
|
||||
canvas.height = Math.max(1, Math.round(height));
|
||||
return canvas;
|
||||
}
|
||||
|
|
@ -5,10 +5,15 @@ import {
|
|||
CanvasSink,
|
||||
type WrappedCanvas,
|
||||
} from "mediabunny";
|
||||
import {
|
||||
type FrameSink,
|
||||
HTMLVideoElementSink,
|
||||
isWebCodecsAvailable,
|
||||
} from "./html-video-element-sink";
|
||||
|
||||
interface VideoSinkData {
|
||||
input: Input;
|
||||
sink: CanvasSink;
|
||||
sink: FrameSink;
|
||||
dispose: () => void;
|
||||
iterator: AsyncGenerator<WrappedCanvas, void, unknown> | null;
|
||||
currentFrame: WrappedCanvas | null;
|
||||
nextFrame: WrappedCanvas | null;
|
||||
|
|
@ -275,7 +280,15 @@ export class VideoCache {
|
|||
|
||||
const canDecode = await videoTrack.canDecode();
|
||||
if (!canDecode) {
|
||||
throw new Error("Video codec not supported for decoding");
|
||||
if (isWebCodecsAvailable()) {
|
||||
throw new Error("Video codec not supported for decoding");
|
||||
}
|
||||
// WebCodecs API absent (e.g. browsers without VideoDecoder support).
|
||||
// Fall back to an HTMLVideoElement-backed sink which uses the
|
||||
// system's native video decoders via the <video> tag.
|
||||
input.dispose();
|
||||
await this.initializeHTMLVideoElementSink({ mediaId, file });
|
||||
return;
|
||||
}
|
||||
|
||||
const sink = new CanvasSink(videoTrack, {
|
||||
|
|
@ -284,8 +297,8 @@ export class VideoCache {
|
|||
});
|
||||
|
||||
this.sinks.set(mediaId, {
|
||||
input,
|
||||
sink,
|
||||
dispose: () => input.dispose(),
|
||||
iterator: null,
|
||||
currentFrame: null,
|
||||
nextFrame: null,
|
||||
|
|
@ -300,6 +313,29 @@ export class VideoCache {
|
|||
}
|
||||
}
|
||||
|
||||
private async initializeHTMLVideoElementSink({
|
||||
mediaId,
|
||||
file,
|
||||
}: {
|
||||
mediaId: string;
|
||||
file: File;
|
||||
}): Promise<void> {
|
||||
const fallbackSink = await HTMLVideoElementSink.create({ file });
|
||||
this.sinks.set(mediaId, {
|
||||
sink: fallbackSink,
|
||||
dispose: () => fallbackSink.dispose(),
|
||||
iterator: null,
|
||||
currentFrame: null,
|
||||
nextFrame: null,
|
||||
lastTime: -1,
|
||||
prefetching: false,
|
||||
prefetchPromise: null,
|
||||
});
|
||||
console.warn(
|
||||
"[VideoCache] WebCodecs unavailable; using HTMLVideoElement fallback for video preview.",
|
||||
);
|
||||
}
|
||||
|
||||
clearVideo({ mediaId }: { mediaId: string }): void {
|
||||
const sinkData = this.sinks.get(mediaId);
|
||||
if (sinkData) {
|
||||
|
|
@ -307,7 +343,7 @@ export class VideoCache {
|
|||
void sinkData.iterator.return();
|
||||
}
|
||||
|
||||
sinkData.input.dispose();
|
||||
sinkData.dispose();
|
||||
this.sinks.delete(mediaId);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue