Merge 6f7dacb4d4 into 238750c025
This commit is contained in:
Andrés Villagrán
GitHub
commit
c924676de6
|
|
@ -0,0 +1,144 @@
|
|||
import { type NextRequest, NextResponse } from "next/server";
|
||||
import { writeFile, readFile, rm, mkdtemp } from "fs/promises";
|
||||
import { spawn } from "child_process";
|
||||
import path from "path";
|
||||
import os from "os";
|
||||
|
||||
const GENERIC_CONVERSION_ERROR = "Video conversion failed.";
|
||||
|
||||
function sanitizeContentDispositionName({
|
||||
rawName,
|
||||
}: {
|
||||
rawName: string;
|
||||
}): string {
|
||||
// Strip characters that can break the Content-Disposition header (quotes,
|
||||
// backslashes, CR/LF, control bytes) and replace non-ASCII so the
|
||||
// "filename=" parameter stays unambiguous across clients.
|
||||
const stripped = rawName
|
||||
// eslint-disable-next-line no-control-regex
|
||||
.replace(/[\\"\r\n\x00-\x1f]/g, "_")
|
||||
.replace(/[^\x20-\x7e]/g, "_")
|
||||
.slice(0, 200);
|
||||
return stripped.length > 0 ? stripped : "video";
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
let workDir: string | null = null;
|
||||
|
||||
try {
|
||||
const formData = await request.formData();
|
||||
const file = formData.get("file") as File | null;
|
||||
|
||||
if (!file || !(file instanceof File)) {
|
||||
return NextResponse.json(
|
||||
{ error: "No file provided" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
workDir = await mkdtemp(path.join(os.tmpdir(), "opencut-convert-"));
|
||||
const ext = path.extname(file.name) || ".mp4";
|
||||
const inputPath = path.join(workDir, `input${ext}`);
|
||||
const outputPath = path.join(workDir, "output.mp4");
|
||||
|
||||
const buffer = Buffer.from(await file.arrayBuffer());
|
||||
await writeFile(inputPath, buffer);
|
||||
|
||||
await runFfmpeg({
|
||||
inputPath,
|
||||
outputPath,
|
||||
});
|
||||
|
||||
const convertedBuffer = await readFile(outputPath);
|
||||
|
||||
const safeName = sanitizeContentDispositionName({
|
||||
rawName: path.basename(file.name, ext),
|
||||
});
|
||||
|
||||
return new Response(new Uint8Array(convertedBuffer), {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": "video/mp4",
|
||||
"Content-Disposition": `attachment; filename="${safeName}.mp4"`,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Video conversion error:", error);
|
||||
return NextResponse.json(
|
||||
{ error: GENERIC_CONVERSION_ERROR },
|
||||
{ status: 500 },
|
||||
);
|
||||
} finally {
|
||||
if (workDir) {
|
||||
await rm(workDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function runFfmpeg({
|
||||
inputPath,
|
||||
outputPath,
|
||||
}: {
|
||||
inputPath: string;
|
||||
outputPath: string;
|
||||
}): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const proc = spawn("ffmpeg", [
|
||||
"-i",
|
||||
inputPath,
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-profile:v",
|
||||
"baseline",
|
||||
"-level",
|
||||
"3.1",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"23",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-g",
|
||||
"60",
|
||||
"-keyint_min",
|
||||
"60",
|
||||
"-sc_threshold",
|
||||
"0",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
"-tag:v",
|
||||
"avc1",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-ac",
|
||||
"2",
|
||||
"-ar",
|
||||
"48000",
|
||||
"-b:a",
|
||||
"128k",
|
||||
"-y",
|
||||
outputPath,
|
||||
]);
|
||||
|
||||
let stderr = "";
|
||||
proc.stderr.on("data", (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
proc.on("close", (code) => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(
|
||||
new Error(
|
||||
`ffmpeg exited with code ${code}. stderr: ${stderr.slice(-500)}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
proc.on("error", (err) => {
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
import { useMemo, useState } from "react";
|
||||
import { TransitionTopIcon } from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import {
|
||||
|
|
@ -34,6 +34,9 @@ import {
|
|||
} from "@/components/section";
|
||||
import { useEditor } from "@/editor/use-editor";
|
||||
import { DEFAULT_EXPORT_OPTIONS } from "@/export/defaults";
|
||||
import { isWebCodecsExportSupported } from "@/services/renderer/media-recorder-support";
|
||||
import { convertVideoToH264 } from "@/media/ffmpeg-convert";
|
||||
import { toast } from "sonner";
|
||||
|
||||
function isExportFormat(value: string): value is ExportFormat {
|
||||
return EXPORT_FORMAT_VALUES.some((formatValue) => formatValue === value);
|
||||
|
|
@ -111,6 +114,20 @@ function ExportPopover({
|
|||
DEFAULT_EXPORT_OPTIONS.includeAudio ?? true,
|
||||
);
|
||||
|
||||
// When the browser lacks WebCodecs encoders, the SceneExporter falls back to
|
||||
// a MediaRecorder-based path that always produces WebM (VP8/VP9 + Opus).
|
||||
// QuickTime and Safari can't play WebM natively, so we offer a one-click
|
||||
// server-side transcode to H.264 MP4 when the user originally asked for MP4.
|
||||
const webCodecsExportSupported = useMemo(
|
||||
() => isWebCodecsExportSupported(),
|
||||
[],
|
||||
);
|
||||
const willFallbackToWebM = !webCodecsExportSupported;
|
||||
const shouldOfferServerTranscode = willFallbackToWebM && format === "mp4";
|
||||
const [transcodeToMp4OnServer, setTranscodeToMp4OnServer] =
|
||||
useState<boolean>(true);
|
||||
const [isTranscoding, setIsTranscoding] = useState(false);
|
||||
|
||||
const handleExport = async () => {
|
||||
if (!activeProject) return;
|
||||
|
||||
|
|
@ -129,10 +146,39 @@ function ExportPopover({
|
|||
}
|
||||
|
||||
if (result.success && result.buffer) {
|
||||
let outBuffer = result.buffer;
|
||||
let outFormat: ExportFormat = format;
|
||||
|
||||
if (shouldOfferServerTranscode) {
|
||||
if (transcodeToMp4OnServer) {
|
||||
setIsTranscoding(true);
|
||||
try {
|
||||
const webmFile = new File([outBuffer], "export.webm", {
|
||||
type: "video/webm",
|
||||
});
|
||||
const mp4File = await convertVideoToH264({ file: webmFile });
|
||||
outBuffer = await mp4File.arrayBuffer();
|
||||
} catch (error) {
|
||||
const message =
|
||||
error instanceof Error ? error.message : "Unknown error";
|
||||
toast.error("Server transcode failed", {
|
||||
description: `${message}. Saving WebM instead.`,
|
||||
});
|
||||
outFormat = "webm";
|
||||
} finally {
|
||||
setIsTranscoding(false);
|
||||
}
|
||||
} else {
|
||||
// User opted out of server transcode: keep WebM extension/mime so
|
||||
// the file matches its actual container.
|
||||
outFormat = "webm";
|
||||
}
|
||||
}
|
||||
|
||||
downloadBuffer({
|
||||
buffer: result.buffer,
|
||||
filename: `${activeProject.metadata.name}${getExportFileExtension({ format })}`,
|
||||
mimeType: getExportMimeType({ format }),
|
||||
buffer: outBuffer,
|
||||
filename: `${activeProject.metadata.name}${getExportFileExtension({ format: outFormat })}`,
|
||||
mimeType: getExportMimeType({ format: outFormat }),
|
||||
});
|
||||
|
||||
editor.project.clearExportState();
|
||||
|
|
@ -250,6 +296,39 @@ function ExportPopover({
|
|||
</div>
|
||||
</SectionContent>
|
||||
</Section>
|
||||
|
||||
{shouldOfferServerTranscode && (
|
||||
<Section collapsible defaultOpen>
|
||||
<SectionHeader>
|
||||
<SectionTitle>Browser compatibility</SectionTitle>
|
||||
</SectionHeader>
|
||||
<SectionContent>
|
||||
<p className="mb-2 text-xs text-amber-600 dark:text-amber-400">
|
||||
This browser can't encode H.264 directly, so the
|
||||
export will produce a WebM file. WebM plays in Chrome,
|
||||
Firefox and VLC, but not natively in QuickTime or
|
||||
Safari.
|
||||
</p>
|
||||
<div className="flex items-start space-x-2">
|
||||
<Checkbox
|
||||
id="transcode-mp4"
|
||||
className="mt-0.5"
|
||||
checked={transcodeToMp4OnServer}
|
||||
onCheckedChange={(checked) =>
|
||||
setTranscodeToMp4OnServer(!!checked)
|
||||
}
|
||||
/>
|
||||
<Label
|
||||
htmlFor="transcode-mp4"
|
||||
className="text-xs leading-snug"
|
||||
>
|
||||
Transcode to H.264 MP4 on the server after export
|
||||
(uploads the WebM and re-encodes via ffmpeg)
|
||||
</Label>
|
||||
</div>
|
||||
</SectionContent>
|
||||
</Section>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="p-3 pt-0">
|
||||
|
|
@ -282,6 +361,18 @@ function ExportPopover({
|
|||
</Button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isTranscoding && !isExporting && (
|
||||
<div className="space-y-2 p-3">
|
||||
<p className="text-muted-foreground text-sm">
|
||||
Transcoding to MP4 on server…
|
||||
</p>
|
||||
<p className="text-muted-foreground text-xs">
|
||||
Uploading the WebM file and re-encoding via ffmpeg. This
|
||||
runs once per export and is independent of preview playback.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,10 @@ import {
|
|||
Input,
|
||||
type WrappedAudioBuffer,
|
||||
} from "mediabunny";
|
||||
import {
|
||||
type AudioBufferSinkLike,
|
||||
FallbackAudioBufferSink,
|
||||
} from "@/services/audio-cache/fallback-audio-buffer-sink";
|
||||
|
||||
export class AudioManager {
|
||||
private audioContext: AudioContext | null = null;
|
||||
|
|
@ -30,7 +34,7 @@ export class AudioManager {
|
|||
private lookaheadSeconds = 2;
|
||||
private scheduleIntervalMs = 500;
|
||||
private clips: AudioClipSource[] = [];
|
||||
private sinks = new Map<string, AudioBufferSink>();
|
||||
private sinks = new Map<string, AudioBufferSinkLike>();
|
||||
private inputs = new Map<string, Input>();
|
||||
private activeClipIds = new Set<string>();
|
||||
private clipIterators = new Map<
|
||||
|
|
@ -267,7 +271,42 @@ export class AudioManager {
|
|||
|
||||
const iterator = sink.buffers(sourceStartTime);
|
||||
this.clipIterators.set(clip.id, iterator);
|
||||
let consecutiveDroppedBufferCount = 0;
|
||||
|
||||
try {
|
||||
await this.consumeClipIterator({
|
||||
clip,
|
||||
clipEnd,
|
||||
iterator,
|
||||
audioContext,
|
||||
sessionId,
|
||||
initialDroppedCount: 0,
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`[AudioManager] clip iterator failed for ${clip.id}; aborting playback for this clip.`,
|
||||
error,
|
||||
);
|
||||
this.clipIterators.delete(clip.id);
|
||||
this.activeClipIds.delete(clip.id);
|
||||
}
|
||||
}
|
||||
|
||||
private async consumeClipIterator({
|
||||
clip,
|
||||
clipEnd,
|
||||
iterator,
|
||||
audioContext,
|
||||
sessionId,
|
||||
initialDroppedCount,
|
||||
}: {
|
||||
clip: AudioClipSource;
|
||||
clipEnd: number;
|
||||
iterator: AsyncGenerator<WrappedAudioBuffer, void, unknown>;
|
||||
audioContext: AudioContext;
|
||||
sessionId: number;
|
||||
initialDroppedCount: number;
|
||||
}): Promise<void> {
|
||||
let consecutiveDroppedBufferCount = initialDroppedCount;
|
||||
|
||||
for await (const { buffer, timestamp } of iterator) {
|
||||
if (!this.editor.playback.getIsPlaying()) return;
|
||||
|
|
@ -605,7 +644,18 @@ export class AudioManager {
|
|||
return null;
|
||||
}
|
||||
|
||||
const sink = new AudioBufferSink(audioTrack);
|
||||
const canDecode = await safeCanDecode({ audioTrack });
|
||||
const sink: AudioBufferSinkLike = canDecode
|
||||
? new AudioBufferSink(audioTrack)
|
||||
: await FallbackAudioBufferSink.create({
|
||||
file: clip.file,
|
||||
audioContext,
|
||||
});
|
||||
if (!canDecode) {
|
||||
console.warn(
|
||||
"[AudioManager] WebCodecs unavailable; decoding clip audio via decodeAudioData.",
|
||||
);
|
||||
}
|
||||
const chunks: AudioBuffer[] = [];
|
||||
let totalSamples = 0;
|
||||
|
||||
|
|
@ -673,12 +723,13 @@ export class AudioManager {
|
|||
clip,
|
||||
}: {
|
||||
clip: AudioClipSource;
|
||||
}): Promise<AudioBufferSink | null> {
|
||||
}): Promise<AudioBufferSinkLike | null> {
|
||||
const existingSink = this.sinks.get(clip.sourceKey);
|
||||
if (existingSink) return existingSink;
|
||||
|
||||
let input: Input | null = null;
|
||||
try {
|
||||
const input = new Input({
|
||||
input = new Input({
|
||||
source: new BlobSource(clip.file),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
|
@ -688,13 +739,47 @@ export class AudioManager {
|
|||
return null;
|
||||
}
|
||||
|
||||
const canDecode = await safeCanDecode({ audioTrack });
|
||||
if (!canDecode) {
|
||||
input.dispose();
|
||||
input = null;
|
||||
const audioContext = this.ensureAudioContext();
|
||||
if (!audioContext) {
|
||||
return null;
|
||||
}
|
||||
const fallback = await FallbackAudioBufferSink.create({
|
||||
file: clip.file,
|
||||
audioContext,
|
||||
});
|
||||
this.sinks.set(clip.sourceKey, fallback);
|
||||
console.warn(
|
||||
"[AudioManager] WebCodecs unavailable; using decodeAudioData fallback for audio clip.",
|
||||
);
|
||||
return fallback;
|
||||
}
|
||||
|
||||
const sink = new AudioBufferSink(audioTrack);
|
||||
this.inputs.set(clip.sourceKey, input);
|
||||
this.sinks.set(clip.sourceKey, sink);
|
||||
return sink;
|
||||
} catch (error) {
|
||||
if (input) {
|
||||
input.dispose();
|
||||
}
|
||||
console.warn("Failed to initialize audio sink:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function safeCanDecode({
|
||||
audioTrack,
|
||||
}: {
|
||||
audioTrack: { canDecode: () => Promise<boolean> };
|
||||
}): Promise<boolean> {
|
||||
try {
|
||||
return await audioTrack.canDecode();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,10 @@ import { canTrackHaveAudio } from "@/timeline";
|
|||
import { mediaSupportsAudio } from "@/media/media-utils";
|
||||
import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/retime";
|
||||
import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny";
|
||||
import {
|
||||
type AudioBufferSinkLike,
|
||||
FallbackAudioBufferSink,
|
||||
} from "@/services/audio-cache/fallback-audio-buffer-sink";
|
||||
import { TICKS_PER_SECOND } from "@/wasm";
|
||||
import {
|
||||
computeRmsBuckets,
|
||||
|
|
@ -271,7 +275,23 @@ async function resolveAudioBufferForAsset({
|
|||
const audioTrack = await input.getPrimaryAudioTrack();
|
||||
if (!audioTrack) return null;
|
||||
|
||||
const sink = new AudioBufferSink(audioTrack);
|
||||
let audioTrackCanDecode = false;
|
||||
try {
|
||||
audioTrackCanDecode = await audioTrack.canDecode();
|
||||
} catch {
|
||||
audioTrackCanDecode = false;
|
||||
}
|
||||
const sink: AudioBufferSinkLike = audioTrackCanDecode
|
||||
? new AudioBufferSink(audioTrack)
|
||||
: await FallbackAudioBufferSink.create({
|
||||
file: asset.file,
|
||||
audioContext,
|
||||
});
|
||||
if (!audioTrackCanDecode) {
|
||||
console.warn(
|
||||
"[audio] WebCodecs unavailable; decoding asset via decodeAudioData fallback.",
|
||||
);
|
||||
}
|
||||
const targetSampleRate = audioContext.sampleRate;
|
||||
|
||||
const chunks: AudioBuffer[] = [];
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
export async function convertVideoToH264({
|
||||
file,
|
||||
}: {
|
||||
file: File;
|
||||
}): Promise<File> {
|
||||
const formData = new FormData();
|
||||
formData.append("file", file);
|
||||
|
||||
const response = await fetch("/api/convert-video", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}));
|
||||
throw new Error(errorData.error || `Conversion failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
const baseName = file.name.replace(/\.[^.]+$/, "");
|
||||
return new File([blob], `${baseName}.mp4`, { type: "video/mp4" });
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ import type { MediaAsset } from "@/media/types";
|
|||
import { readVideoFile } from "./mediabunny";
|
||||
import type { VideoFileData } from "./mediabunny";
|
||||
import { renderThumbnailDataUrl } from "./thumbnail";
|
||||
import { convertVideoToH264 } from "./ffmpeg-convert";
|
||||
|
||||
export interface ProcessedMediaAsset extends Omit<MediaAsset, "id"> {}
|
||||
|
||||
|
|
@ -132,8 +133,49 @@ export async function processMediaAssets({
|
|||
width = result.width;
|
||||
height = result.height;
|
||||
} else if (fileType === "video") {
|
||||
let currentFile = file;
|
||||
let currentUrl = url;
|
||||
try {
|
||||
const videoData = await readVideoFile({ file });
|
||||
let videoData = await readVideoFile({ file: currentFile });
|
||||
|
||||
if (!videoData.canDecode) {
|
||||
toast.info(`Converting ${file.name} to a compatible format…`);
|
||||
const convertedFile = await convertVideoToH264({
|
||||
file: currentFile,
|
||||
});
|
||||
|
||||
const convertedStorageCheck = await storageService.canStoreFile({
|
||||
size: convertedFile.size,
|
||||
});
|
||||
if (!convertedStorageCheck.canStore) {
|
||||
throw new Error(
|
||||
getStorageLimitDescription({
|
||||
fileSize: convertedFile.size,
|
||||
availableBytes: convertedStorageCheck.availableBytes,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
URL.revokeObjectURL(currentUrl);
|
||||
currentFile = convertedFile;
|
||||
currentUrl = URL.createObjectURL(currentFile);
|
||||
videoData = await readVideoFile({ file: currentFile });
|
||||
|
||||
if (!videoData.canDecode) {
|
||||
// Conversion produced a file the browser still can't decode
|
||||
// via mediabunny/WebCodecs. The HTMLVideoElement fallback in
|
||||
// VideoCache will handle preview rendering, so warn but
|
||||
// continue importing.
|
||||
toast.warning(`${file.name} converted, preview may be limited`, {
|
||||
description: getUnsupportedVideoDescription({
|
||||
codec: videoData.codec,
|
||||
}),
|
||||
});
|
||||
} else {
|
||||
toast.success(`${file.name} converted successfully`);
|
||||
}
|
||||
}
|
||||
|
||||
duration = videoData.duration;
|
||||
width = videoData.width;
|
||||
height = videoData.height;
|
||||
|
|
@ -143,13 +185,27 @@ export async function processMediaAssets({
|
|||
hasAudio = videoData.hasAudio;
|
||||
thumbnailUrl = videoData.thumbnailUrl ?? undefined;
|
||||
|
||||
if (!videoData.canDecode) {
|
||||
toast.error(`Can't preview ${file.name}`, {
|
||||
description: getUnsupportedVideoDescription({
|
||||
codec: videoData.codec,
|
||||
}),
|
||||
});
|
||||
processedAssets.push({
|
||||
name: currentFile.name,
|
||||
type: fileType,
|
||||
file: currentFile,
|
||||
url: currentUrl,
|
||||
thumbnailUrl,
|
||||
duration,
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
hasAudio,
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
completed += 1;
|
||||
if (onProgress) {
|
||||
const percent = Math.round((completed / total) * 100);
|
||||
onProgress({ progress: percent });
|
||||
}
|
||||
continue;
|
||||
} catch (error) {
|
||||
const message =
|
||||
error instanceof Error
|
||||
|
|
@ -159,6 +215,8 @@ export async function processMediaAssets({
|
|||
toast.error(`Couldn't process ${file.name}`, {
|
||||
description: message,
|
||||
});
|
||||
URL.revokeObjectURL(currentUrl);
|
||||
continue;
|
||||
}
|
||||
} else if (fileType === "audio") {
|
||||
duration = await getMediaDuration({ file });
|
||||
|
|
|
|||
|
|
@ -0,0 +1,308 @@
|
|||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
test,
|
||||
} from "bun:test";
|
||||
import {
|
||||
FallbackAudioBufferSink,
|
||||
isWebAudioDecoderAvailable,
|
||||
sliceAudioBuffer,
|
||||
} from "../fallback-audio-buffer-sink";
|
||||
|
||||
type AudioBufferLike = {
|
||||
length: number;
|
||||
sampleRate: number;
|
||||
numberOfChannels: number;
|
||||
duration: number;
|
||||
getChannelData: (channel: number) => Float32Array;
|
||||
copyToChannel: (
|
||||
source: Float32Array,
|
||||
channel: number,
|
||||
startInChannel?: number,
|
||||
) => void;
|
||||
};
|
||||
|
||||
function createTestAudioBuffer({
|
||||
durationSeconds = 1,
|
||||
sampleRate = 8,
|
||||
channelCount = 2,
|
||||
}: {
|
||||
durationSeconds?: number;
|
||||
sampleRate?: number;
|
||||
channelCount?: number;
|
||||
} = {}): AudioBufferLike {
|
||||
const length = Math.round(durationSeconds * sampleRate);
|
||||
const channels = Array.from(
|
||||
{ length: channelCount },
|
||||
(_, channelIndex) =>
|
||||
new Float32Array(
|
||||
Array.from({ length }, (_, i) => channelIndex + 1 + i * 0.01),
|
||||
),
|
||||
);
|
||||
return {
|
||||
length,
|
||||
sampleRate,
|
||||
numberOfChannels: channelCount,
|
||||
duration: durationSeconds,
|
||||
getChannelData(channel: number) {
|
||||
return channels[channel];
|
||||
},
|
||||
// eslint-disable-next-line opencut/prefer-object-params -- mirrors the AudioBuffer.copyToChannel Web API signature so this fake behaves like the real one
|
||||
copyToChannel(
|
||||
source: Float32Array,
|
||||
channel: number,
|
||||
startInChannel = 0,
|
||||
) {
|
||||
channels[channel].set(source, startInChannel);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function installFakeAudioBuffer(): { restore: () => void } {
|
||||
const root = globalThis as { AudioBuffer?: unknown };
|
||||
const original = root.AudioBuffer;
|
||||
class FakeAudioBuffer implements AudioBufferLike {
|
||||
length: number;
|
||||
sampleRate: number;
|
||||
numberOfChannels: number;
|
||||
duration: number;
|
||||
private channels: Float32Array[];
|
||||
|
||||
constructor(opts: {
|
||||
length: number;
|
||||
numberOfChannels: number;
|
||||
sampleRate: number;
|
||||
}) {
|
||||
this.length = opts.length;
|
||||
this.sampleRate = opts.sampleRate;
|
||||
this.numberOfChannels = opts.numberOfChannels;
|
||||
this.duration = opts.length / opts.sampleRate;
|
||||
this.channels = Array.from(
|
||||
{ length: opts.numberOfChannels },
|
||||
() => new Float32Array(opts.length),
|
||||
);
|
||||
}
|
||||
|
||||
getChannelData(channel: number): Float32Array {
|
||||
return this.channels[channel];
|
||||
}
|
||||
|
||||
// eslint-disable-next-line opencut/prefer-object-params -- mirrors AudioBuffer.copyToChannel Web API signature
|
||||
copyToChannel(
|
||||
source: Float32Array,
|
||||
channel: number,
|
||||
startInChannel = 0,
|
||||
): void {
|
||||
this.channels[channel].set(source, startInChannel);
|
||||
}
|
||||
}
|
||||
root.AudioBuffer = FakeAudioBuffer;
|
||||
return {
|
||||
restore() {
|
||||
if (original === undefined) {
|
||||
delete root.AudioBuffer;
|
||||
} else {
|
||||
root.AudioBuffer = original;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("isWebAudioDecoderAvailable", () => {
|
||||
const root = globalThis as { AudioDecoder?: unknown };
|
||||
let original: unknown;
|
||||
|
||||
beforeEach(() => {
|
||||
original = root.AudioDecoder;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (original === undefined) {
|
||||
delete root.AudioDecoder;
|
||||
} else {
|
||||
root.AudioDecoder = original;
|
||||
}
|
||||
});
|
||||
|
||||
test("returns false when AudioDecoder undefined", () => {
|
||||
delete root.AudioDecoder;
|
||||
expect(isWebAudioDecoderAvailable()).toBe(false);
|
||||
});
|
||||
|
||||
test("returns true when AudioDecoder defined", () => {
|
||||
root.AudioDecoder = class {};
|
||||
expect(isWebAudioDecoderAvailable()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sliceAudioBuffer", () => {
|
||||
let fakeAudioBuffer: ReturnType<typeof installFakeAudioBuffer>;
|
||||
|
||||
beforeEach(() => {
|
||||
fakeAudioBuffer = installFakeAudioBuffer();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fakeAudioBuffer.restore();
|
||||
});
|
||||
|
||||
test("slices to the requested range", () => {
|
||||
const buffer = createTestAudioBuffer({
|
||||
durationSeconds: 1,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sliced = sliceAudioBuffer({
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
buffer: buffer as unknown as AudioBuffer,
|
||||
start: 0.25,
|
||||
end: 0.75,
|
||||
});
|
||||
expect(sliced.length).toBe(4);
|
||||
expect(sliced.numberOfChannels).toBe(1);
|
||||
expect(sliced.sampleRate).toBe(8);
|
||||
const channel = sliced.getChannelData(0);
|
||||
expect(channel[0]).toBeCloseTo(1 + 0.02, 5);
|
||||
expect(channel[3]).toBeCloseTo(1 + 0.05, 5);
|
||||
});
|
||||
|
||||
test("clamps out-of-range start/end", () => {
|
||||
const buffer = createTestAudioBuffer({
|
||||
durationSeconds: 0.5,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sliced = sliceAudioBuffer({
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
buffer: buffer as unknown as AudioBuffer,
|
||||
start: -1,
|
||||
end: 5,
|
||||
});
|
||||
expect(sliced.length).toBe(buffer.length);
|
||||
});
|
||||
|
||||
test("preserves channel count", () => {
|
||||
const buffer = createTestAudioBuffer({
|
||||
durationSeconds: 1,
|
||||
sampleRate: 8,
|
||||
channelCount: 2,
|
||||
});
|
||||
const sliced = sliceAudioBuffer({
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
buffer: buffer as unknown as AudioBuffer,
|
||||
start: 0,
|
||||
end: 0.5,
|
||||
});
|
||||
expect(sliced.numberOfChannels).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("FallbackAudioBufferSink.buffers", () => {
|
||||
let fakeAudioBuffer: ReturnType<typeof installFakeAudioBuffer>;
|
||||
|
||||
beforeEach(() => {
|
||||
fakeAudioBuffer = installFakeAudioBuffer();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fakeAudioBuffer.restore();
|
||||
});
|
||||
|
||||
test("yields a single buffer covering the full range by default", async () => {
|
||||
const decoded = createTestAudioBuffer({
|
||||
durationSeconds: 1,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sink = new FallbackAudioBufferSink(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
decoded as unknown as AudioBuffer,
|
||||
);
|
||||
const yielded = [];
|
||||
for await (const wrapped of sink.buffers()) {
|
||||
yielded.push(wrapped);
|
||||
}
|
||||
expect(yielded).toHaveLength(1);
|
||||
expect(yielded[0].timestamp).toBe(0);
|
||||
expect(yielded[0].duration).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
test("yields a slice when startTimestamp/endTimestamp provided", async () => {
|
||||
const decoded = createTestAudioBuffer({
|
||||
durationSeconds: 2,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sink = new FallbackAudioBufferSink(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
decoded as unknown as AudioBuffer,
|
||||
);
|
||||
const yielded = [];
|
||||
for await (const wrapped of sink.buffers(0.5, 1.5)) {
|
||||
yielded.push(wrapped);
|
||||
}
|
||||
expect(yielded).toHaveLength(1);
|
||||
expect(yielded[0].timestamp).toBe(0.5);
|
||||
expect(yielded[0].duration).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
test("treats NaN/Infinity startTimestamp as 0", async () => {
|
||||
const decoded = createTestAudioBuffer({
|
||||
durationSeconds: 1,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sink = new FallbackAudioBufferSink(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
decoded as unknown as AudioBuffer,
|
||||
);
|
||||
const yielded = [];
|
||||
for await (const wrapped of sink.buffers(Number.NaN)) {
|
||||
yielded.push(wrapped);
|
||||
}
|
||||
expect(yielded).toHaveLength(1);
|
||||
expect(yielded[0].timestamp).toBe(0);
|
||||
expect(yielded[0].duration).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
test("yields nothing when start >= end (or beyond duration)", async () => {
|
||||
const decoded = createTestAudioBuffer({
|
||||
durationSeconds: 1,
|
||||
sampleRate: 8,
|
||||
channelCount: 1,
|
||||
});
|
||||
const sink = new FallbackAudioBufferSink(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
decoded as unknown as AudioBuffer,
|
||||
);
|
||||
|
||||
const yielded = [];
|
||||
for await (const wrapped of sink.buffers(2)) {
|
||||
yielded.push(wrapped);
|
||||
}
|
||||
expect(yielded).toHaveLength(0);
|
||||
|
||||
const yielded2 = [];
|
||||
for await (const wrapped of sink.buffers(0.4, 0.4)) {
|
||||
yielded2.push(wrapped);
|
||||
}
|
||||
expect(yielded2).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("exposes sampleRate/numberOfChannels/duration getters", () => {
|
||||
const decoded = createTestAudioBuffer({
|
||||
durationSeconds: 3,
|
||||
sampleRate: 16,
|
||||
channelCount: 2,
|
||||
});
|
||||
const sink = new FallbackAudioBufferSink(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test
|
||||
decoded as unknown as AudioBuffer,
|
||||
);
|
||||
expect(sink.sampleRate).toBe(16);
|
||||
expect(sink.numberOfChannels).toBe(2);
|
||||
expect(sink.duration).toBe(3);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
import type { WrappedAudioBuffer } from "mediabunny";
|
||||
|
||||
export interface AudioBufferSinkLike {
|
||||
buffers(
|
||||
startTimestamp?: number,
|
||||
endTimestamp?: number,
|
||||
): AsyncGenerator<WrappedAudioBuffer, void, unknown>;
|
||||
}
|
||||
|
||||
export function isWebAudioDecoderAvailable(): boolean {
|
||||
return (
|
||||
typeof (globalThis as { AudioDecoder?: unknown }).AudioDecoder !==
|
||||
"undefined"
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* AudioBufferSink implementation backed by AudioContext.decodeAudioData.
|
||||
*
|
||||
* Used when mediabunny's AudioSampleSink cannot decode the track because the
|
||||
* environment lacks the WebCodecs AudioDecoder API. AudioContext.decodeAudioData
|
||||
* uses the system's native audio decoders, mirroring the strategy used by
|
||||
* HTMLVideoElementSink for video. The entire compressed audio is decoded
|
||||
* upfront and exposed through a buffers() iterator that yields a single
|
||||
* AudioBuffer slice covering the requested range.
|
||||
*/
|
||||
export class FallbackAudioBufferSink implements AudioBufferSinkLike {
|
||||
constructor(private readonly decoded: AudioBuffer) {}
|
||||
|
||||
static async create({
|
||||
file,
|
||||
audioContext,
|
||||
}: {
|
||||
file: File;
|
||||
audioContext: BaseAudioContext;
|
||||
}): Promise<FallbackAudioBufferSink> {
|
||||
const arrayBuffer = await file.arrayBuffer();
|
||||
const decoded = await decodeArrayBuffer({ arrayBuffer, audioContext });
|
||||
return new FallbackAudioBufferSink(decoded);
|
||||
}
|
||||
|
||||
get sampleRate(): number {
|
||||
return this.decoded.sampleRate;
|
||||
}
|
||||
|
||||
get numberOfChannels(): number {
|
||||
return this.decoded.numberOfChannels;
|
||||
}
|
||||
|
||||
get duration(): number {
|
||||
return this.decoded.duration;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line opencut/prefer-object-params -- mirrors mediabunny's AudioBufferSink.buffers(start, end) so consumers can swap sinks transparently
|
||||
async *buffers(
|
||||
startTimestamp = 0,
|
||||
endTimestamp?: number,
|
||||
): AsyncGenerator<WrappedAudioBuffer, void, unknown> {
|
||||
const totalDuration = this.decoded.duration;
|
||||
const safeStart = Number.isFinite(startTimestamp) ? startTimestamp : 0;
|
||||
const start = Math.max(0, Math.min(safeStart, totalDuration));
|
||||
const end =
|
||||
typeof endTimestamp === "number" && Number.isFinite(endTimestamp)
|
||||
? Math.min(endTimestamp, totalDuration)
|
||||
: totalDuration;
|
||||
if (start >= end) return;
|
||||
|
||||
const sliced = sliceAudioBuffer({
|
||||
buffer: this.decoded,
|
||||
start,
|
||||
end,
|
||||
});
|
||||
|
||||
yield {
|
||||
buffer: sliced,
|
||||
timestamp: start,
|
||||
duration: sliced.duration,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function decodeArrayBuffer({
|
||||
arrayBuffer,
|
||||
audioContext,
|
||||
}: {
|
||||
arrayBuffer: ArrayBuffer;
|
||||
audioContext: BaseAudioContext;
|
||||
}): Promise<AudioBuffer> {
|
||||
return new Promise<AudioBuffer>((resolve, reject) => {
|
||||
audioContext.decodeAudioData(
|
||||
arrayBuffer,
|
||||
(buffer) => resolve(buffer),
|
||||
(err) =>
|
||||
reject(
|
||||
err instanceof Error
|
||||
? err
|
||||
: new Error(`decodeAudioData failed: ${String(err)}`),
|
||||
),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export function sliceAudioBuffer({
|
||||
buffer,
|
||||
start,
|
||||
end,
|
||||
}: {
|
||||
buffer: AudioBuffer;
|
||||
start: number;
|
||||
end: number;
|
||||
}): AudioBuffer {
|
||||
const sampleRate = buffer.sampleRate;
|
||||
const startSample = Math.max(
|
||||
0,
|
||||
Math.min(buffer.length, Math.floor(start * sampleRate)),
|
||||
);
|
||||
const endSample = Math.max(
|
||||
startSample,
|
||||
Math.min(buffer.length, Math.ceil(end * sampleRate)),
|
||||
);
|
||||
const length = Math.max(1, endSample - startSample);
|
||||
|
||||
const target = new AudioBuffer({
|
||||
length,
|
||||
numberOfChannels: Math.max(1, buffer.numberOfChannels),
|
||||
sampleRate,
|
||||
});
|
||||
for (let channel = 0; channel < buffer.numberOfChannels; channel++) {
|
||||
const channelView = buffer
|
||||
.getChannelData(channel)
|
||||
.subarray(startSample, endSample);
|
||||
target.copyToChannel(channelView, channel, 0);
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
||||
import {
|
||||
detectMediaRecorderSupport,
|
||||
isWebCodecsExportSupported,
|
||||
} from "../media-recorder-support";
|
||||
|
||||
describe("isWebCodecsExportSupported", () => {
|
||||
const root = globalThis as {
|
||||
VideoEncoder?: unknown;
|
||||
AudioEncoder?: unknown;
|
||||
};
|
||||
let originalVideoEncoder: unknown;
|
||||
let originalAudioEncoder: unknown;
|
||||
|
||||
beforeEach(() => {
|
||||
originalVideoEncoder = root.VideoEncoder;
|
||||
originalAudioEncoder = root.AudioEncoder;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (originalVideoEncoder === undefined) delete root.VideoEncoder;
|
||||
else root.VideoEncoder = originalVideoEncoder;
|
||||
if (originalAudioEncoder === undefined) delete root.AudioEncoder;
|
||||
else root.AudioEncoder = originalAudioEncoder;
|
||||
});
|
||||
|
||||
test("false when either WebCodecs encoder is missing", () => {
|
||||
delete root.VideoEncoder;
|
||||
delete root.AudioEncoder;
|
||||
expect(isWebCodecsExportSupported()).toBe(false);
|
||||
|
||||
root.VideoEncoder = class {};
|
||||
delete root.AudioEncoder;
|
||||
expect(isWebCodecsExportSupported()).toBe(false);
|
||||
|
||||
delete root.VideoEncoder;
|
||||
root.AudioEncoder = class {};
|
||||
expect(isWebCodecsExportSupported()).toBe(false);
|
||||
});
|
||||
|
||||
test("true only when both encoders are defined", () => {
|
||||
root.VideoEncoder = class {};
|
||||
root.AudioEncoder = class {};
|
||||
expect(isWebCodecsExportSupported()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectMediaRecorderSupport", () => {
|
||||
const root = globalThis as {
|
||||
MediaRecorder?: unknown;
|
||||
};
|
||||
let originalMediaRecorder: unknown;
|
||||
|
||||
beforeEach(() => {
|
||||
originalMediaRecorder = root.MediaRecorder;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (originalMediaRecorder === undefined) delete root.MediaRecorder;
|
||||
else root.MediaRecorder = originalMediaRecorder;
|
||||
});
|
||||
|
||||
test("unsupported when MediaRecorder is undefined", () => {
|
||||
delete root.MediaRecorder;
|
||||
expect(detectMediaRecorderSupport()).toEqual({
|
||||
supported: false,
|
||||
mimeType: null,
|
||||
});
|
||||
});
|
||||
|
||||
test("returns the first supported mime type", () => {
|
||||
class FakeMediaRecorder {
|
||||
static isTypeSupported(mimeType: string): boolean {
|
||||
return mimeType === "video/webm;codecs=vp8,opus";
|
||||
}
|
||||
}
|
||||
root.MediaRecorder = FakeMediaRecorder;
|
||||
expect(detectMediaRecorderSupport()).toEqual({
|
||||
supported: true,
|
||||
mimeType: "video/webm;codecs=vp8,opus",
|
||||
});
|
||||
});
|
||||
|
||||
test("unsupported when no candidate mime type matches", () => {
|
||||
class FakeMediaRecorder {
|
||||
static isTypeSupported(): boolean {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
root.MediaRecorder = FakeMediaRecorder;
|
||||
expect(detectMediaRecorderSupport()).toEqual({
|
||||
supported: false,
|
||||
mimeType: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,195 @@
|
|||
import type { FrameRate } from "opencut-wasm";
|
||||
import { TICKS_PER_SECOND } from "@/wasm";
|
||||
import { frameRateToFloat } from "@/fps/utils";
|
||||
import type { RootNode } from "./nodes/root-node";
|
||||
import type { CanvasRenderer } from "./canvas-renderer";
|
||||
import { detectMediaRecorderSupport } from "./media-recorder-support";
|
||||
|
||||
export {
|
||||
detectMediaRecorderSupport,
|
||||
isWebCodecsExportSupported,
|
||||
} from "./media-recorder-support";
|
||||
export type { MediaRecorderSupport } from "./media-recorder-support";
|
||||
|
||||
/**
|
||||
* MediaRecorder-based exporter used when WebCodecs encoders
|
||||
* (VideoEncoder / AudioEncoder) are not available in the current browser.
|
||||
*
|
||||
* Captures the renderer's output canvas via `canvas.captureStream(fps)` and
|
||||
* mixes the project audio buffer through a `MediaStreamAudioDestinationNode`.
|
||||
* The combined MediaStream is recorded with `MediaRecorder`, producing a
|
||||
* WebM blob (which `<video>` can play natively on the same browser without
|
||||
* WebCodecs).
|
||||
*
|
||||
* Trade-offs vs. the WebCodecs path:
|
||||
* - Output is always WebM (most browsers don't expose an MP4 muxer in
|
||||
* MediaRecorder). The caller's requested format is honoured best-effort
|
||||
* but a WebM container is the realistic floor.
|
||||
* - The renderer is driven at real time, so a 60-second timeline takes
|
||||
* ~60 seconds to export.
|
||||
*/
|
||||
export interface MediaRecorderExportParams {
|
||||
renderer: CanvasRenderer;
|
||||
rootNode: RootNode;
|
||||
fps: FrameRate;
|
||||
bitrate: number;
|
||||
audioBuffer?: AudioBuffer | null;
|
||||
onProgress?: (progress: number) => void;
|
||||
signal?: { cancelled: boolean };
|
||||
}
|
||||
|
||||
export async function exportWithMediaRecorder({
|
||||
renderer,
|
||||
rootNode,
|
||||
fps,
|
||||
bitrate,
|
||||
audioBuffer,
|
||||
onProgress,
|
||||
signal,
|
||||
}: MediaRecorderExportParams): Promise<ArrayBuffer | null> {
|
||||
const support = detectMediaRecorderSupport();
|
||||
if (!support.supported || !support.mimeType) {
|
||||
throw new Error(
|
||||
"MediaRecorder is not available in this browser, so export is not supported.",
|
||||
);
|
||||
}
|
||||
|
||||
const fpsFloat = frameRateToFloat(fps);
|
||||
const ticksPerFrame = Math.round(
|
||||
(TICKS_PER_SECOND * fps.denominator) / fps.numerator,
|
||||
);
|
||||
const frameCount = Math.max(1, Math.floor(rootNode.duration / ticksPerFrame));
|
||||
|
||||
const canvas = renderer.getOutputCanvas();
|
||||
const videoStream = canvas.captureStream(fpsFloat);
|
||||
const combinedStream = new MediaStream();
|
||||
for (const track of videoStream.getVideoTracks()) {
|
||||
combinedStream.addTrack(track);
|
||||
}
|
||||
|
||||
const audioGraph = audioBuffer
|
||||
? createAudioGraphForStream({ audioBuffer })
|
||||
: null;
|
||||
if (audioGraph) {
|
||||
for (const track of audioGraph.stream.getAudioTracks()) {
|
||||
combinedStream.addTrack(track);
|
||||
}
|
||||
}
|
||||
|
||||
const recorder = new MediaRecorder(combinedStream, {
|
||||
mimeType: support.mimeType,
|
||||
videoBitsPerSecond: bitrate,
|
||||
});
|
||||
|
||||
const chunks: Blob[] = [];
|
||||
recorder.addEventListener("dataavailable", (event) => {
|
||||
if (event.data && event.data.size > 0) {
|
||||
chunks.push(event.data);
|
||||
}
|
||||
});
|
||||
|
||||
const recordingFinished = new Promise<Blob>((resolve, reject) => {
|
||||
recorder.addEventListener("stop", () => {
|
||||
resolve(new Blob(chunks, { type: support.mimeType ?? "video/webm" }));
|
||||
});
|
||||
recorder.addEventListener("error", () => {
|
||||
reject(new Error("MediaRecorder failed during export."));
|
||||
});
|
||||
});
|
||||
|
||||
try {
|
||||
recorder.start();
|
||||
|
||||
// Kick off audio playback into the destination node at the same wall-clock
|
||||
// moment as the renderer loop so the captured stream is aligned.
|
||||
if (audioGraph) {
|
||||
audioGraph.start();
|
||||
}
|
||||
|
||||
const startedAt = performance.now();
|
||||
for (let frame = 0; frame < frameCount; frame++) {
|
||||
if (signal?.cancelled) {
|
||||
recorder.stop();
|
||||
await recordingFinished.catch(() => null);
|
||||
return null;
|
||||
}
|
||||
|
||||
const targetWallTimeMs = startedAt + (frame * 1000) / fpsFloat;
|
||||
const waitMs = targetWallTimeMs - performance.now();
|
||||
if (waitMs > 0) {
|
||||
await sleep({ ms: waitMs });
|
||||
}
|
||||
|
||||
const timeTicks = frame * ticksPerFrame;
|
||||
await renderer.render({ node: rootNode, time: timeTicks });
|
||||
|
||||
onProgress?.(frame / frameCount);
|
||||
}
|
||||
|
||||
// Give the recorder a moment to flush the last frames after rendering ends.
|
||||
await sleep({ ms: 250 });
|
||||
recorder.stop();
|
||||
onProgress?.(1);
|
||||
|
||||
const blob = await recordingFinished;
|
||||
if (!blob) return null;
|
||||
return await blob.arrayBuffer();
|
||||
} finally {
|
||||
if (audioGraph) {
|
||||
audioGraph.dispose();
|
||||
}
|
||||
for (const track of combinedStream.getTracks()) {
|
||||
track.stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface AudioGraph {
|
||||
stream: MediaStream;
|
||||
start: () => void;
|
||||
dispose: () => void;
|
||||
}
|
||||
|
||||
function createAudioGraphForStream({
|
||||
audioBuffer,
|
||||
}: {
|
||||
audioBuffer: AudioBuffer;
|
||||
}): AudioGraph {
|
||||
const audioContext = new AudioContext({
|
||||
sampleRate: audioBuffer.sampleRate,
|
||||
});
|
||||
const destination = audioContext.createMediaStreamDestination();
|
||||
const source = audioContext.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(destination);
|
||||
|
||||
let started = false;
|
||||
let disposed = false;
|
||||
|
||||
return {
|
||||
stream: destination.stream,
|
||||
start() {
|
||||
if (started || disposed) return;
|
||||
started = true;
|
||||
source.start(0);
|
||||
},
|
||||
dispose() {
|
||||
if (disposed) return;
|
||||
disposed = true;
|
||||
try {
|
||||
source.stop();
|
||||
} catch {
|
||||
// already stopped
|
||||
}
|
||||
source.disconnect();
|
||||
destination.disconnect();
|
||||
void audioContext.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function sleep({ ms }: { ms: number }): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
/**
|
||||
* Lightweight feature-detection helpers for the MediaRecorder-based export
|
||||
* fallback. Kept dependency-free so unit tests don't drag in WASM modules.
|
||||
*/
|
||||
|
||||
export interface MediaRecorderSupport {
|
||||
supported: boolean;
|
||||
mimeType: string | null;
|
||||
}
|
||||
|
||||
const MEDIA_RECORDER_CANDIDATE_MIME_TYPES = [
|
||||
"video/webm;codecs=vp9,opus",
|
||||
"video/webm;codecs=vp8,opus",
|
||||
"video/webm",
|
||||
] as const;
|
||||
|
||||
export function detectMediaRecorderSupport(): MediaRecorderSupport {
|
||||
if (typeof MediaRecorder === "undefined") {
|
||||
return { supported: false, mimeType: null };
|
||||
}
|
||||
for (const candidate of MEDIA_RECORDER_CANDIDATE_MIME_TYPES) {
|
||||
if (MediaRecorder.isTypeSupported(candidate)) {
|
||||
return { supported: true, mimeType: candidate };
|
||||
}
|
||||
}
|
||||
return { supported: false, mimeType: null };
|
||||
}
|
||||
|
||||
export function isWebCodecsExportSupported(): boolean {
|
||||
const root = globalThis as {
|
||||
VideoEncoder?: unknown;
|
||||
AudioEncoder?: unknown;
|
||||
};
|
||||
return (
|
||||
typeof root.VideoEncoder !== "undefined" &&
|
||||
typeof root.AudioEncoder !== "undefined"
|
||||
);
|
||||
}
|
||||
|
|
@ -19,6 +19,10 @@ import { frameRateToFloat } from "@/fps/utils";
|
|||
import type { RootNode } from "./nodes/root-node";
|
||||
import type { ExportFormat, ExportQuality } from "@/export";
|
||||
import { CanvasRenderer } from "./canvas-renderer";
|
||||
import {
|
||||
exportWithMediaRecorder,
|
||||
isWebCodecsExportSupported,
|
||||
} from "./media-recorder-exporter";
|
||||
|
||||
type ExportParams = {
|
||||
width: number;
|
||||
|
|
@ -37,6 +41,15 @@ const qualityMap = {
|
|||
very_high: QUALITY_VERY_HIGH,
|
||||
};
|
||||
|
||||
// Numeric bitrate (bits-per-second) used by the MediaRecorder export path,
|
||||
// which doesn't accept mediabunny's opaque `Quality` presets.
|
||||
const mediaRecorderBitrateMap: Record<ExportQuality, number> = {
|
||||
low: 2_500_000,
|
||||
medium: 5_000_000,
|
||||
high: 10_000_000,
|
||||
very_high: 20_000_000,
|
||||
};
|
||||
|
||||
export type SceneExporterEvents = {
|
||||
progress: [progress: number];
|
||||
complete: [buffer: ArrayBuffer];
|
||||
|
|
@ -84,6 +97,10 @@ export class SceneExporter extends EventEmitter<SceneExporterEvents> {
|
|||
}: {
|
||||
rootNode: RootNode;
|
||||
}): Promise<ArrayBuffer | null> {
|
||||
if (!isWebCodecsExportSupported()) {
|
||||
return this.exportViaMediaRecorder({ rootNode });
|
||||
}
|
||||
|
||||
const fps = this.renderer.fps;
|
||||
const fpsFloat = frameRateToFloat(fps);
|
||||
const ticksPerFrame = Math.round(
|
||||
|
|
@ -168,4 +185,56 @@ export class SceneExporter extends EventEmitter<SceneExporterEvents> {
|
|||
this.emit("complete", buffer);
|
||||
return buffer;
|
||||
}
|
||||
|
||||
private async exportViaMediaRecorder({
|
||||
rootNode,
|
||||
}: {
|
||||
rootNode: RootNode;
|
||||
}): Promise<ArrayBuffer | null> {
|
||||
try {
|
||||
const cancellationSignal = { cancelled: false };
|
||||
const onTickCancel = () => {
|
||||
if (this.isCancelled) {
|
||||
cancellationSignal.cancelled = true;
|
||||
}
|
||||
};
|
||||
|
||||
const buffer = await exportWithMediaRecorder({
|
||||
renderer: this.renderer,
|
||||
rootNode,
|
||||
fps: this.renderer.fps,
|
||||
bitrate: mediaRecorderBitrateMap[this.quality],
|
||||
audioBuffer: this.shouldIncludeAudio ? this.audioBuffer : null,
|
||||
onProgress: (progress) => {
|
||||
onTickCancel();
|
||||
this.emit("progress", progress);
|
||||
},
|
||||
signal: cancellationSignal,
|
||||
});
|
||||
|
||||
if (cancellationSignal.cancelled) {
|
||||
this.emit("cancelled");
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!buffer) {
|
||||
this.emit("error", new Error("Failed to export video"));
|
||||
return null;
|
||||
}
|
||||
|
||||
this.emit("complete", buffer);
|
||||
return buffer;
|
||||
} catch (error) {
|
||||
const err =
|
||||
error instanceof Error
|
||||
? error
|
||||
: new Error(
|
||||
typeof error === "string"
|
||||
? error
|
||||
: "MediaRecorder export failed",
|
||||
);
|
||||
this.emit("error", err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,259 @@
|
|||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
test,
|
||||
} from "bun:test";
|
||||
import {
|
||||
clampFallbackFps,
|
||||
HTMLVideoElementSink,
|
||||
isWebCodecsAvailable,
|
||||
type VideoElementHandle,
|
||||
} from "../html-video-element-sink";
|
||||
|
||||
type FakeCanvas = {
|
||||
width: number;
|
||||
height: number;
|
||||
getContext: (kind: string) => FakeCanvasContext | null;
|
||||
};
|
||||
|
||||
type FakeCanvasContext = {
|
||||
drawImage: (...args: unknown[]) => void;
|
||||
drawnCount: number;
|
||||
};
|
||||
|
||||
function installFakeDom(): {
|
||||
createdCanvases: FakeCanvas[];
|
||||
restore: () => void;
|
||||
} {
|
||||
const createdCanvases: FakeCanvas[] = [];
|
||||
const originalDocument = (
|
||||
globalThis as { document?: unknown }
|
||||
).document;
|
||||
|
||||
const fakeDocument = {
|
||||
createElement(tag: string) {
|
||||
if (tag !== "canvas") {
|
||||
throw new Error(`unexpected createElement(${tag}) in test`);
|
||||
}
|
||||
const ctx: FakeCanvasContext = {
|
||||
drawnCount: 0,
|
||||
drawImage() {
|
||||
ctx.drawnCount += 1;
|
||||
},
|
||||
};
|
||||
const canvas: FakeCanvas = {
|
||||
width: 0,
|
||||
height: 0,
|
||||
getContext(kind: string) {
|
||||
return kind === "2d" ? ctx : null;
|
||||
},
|
||||
};
|
||||
createdCanvases.push(canvas);
|
||||
return canvas;
|
||||
},
|
||||
};
|
||||
|
||||
(globalThis as { document?: unknown }).document = fakeDocument;
|
||||
|
||||
return {
|
||||
createdCanvases,
|
||||
restore() {
|
||||
(globalThis as { document?: unknown }).document = originalDocument;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createFakeHandle({
|
||||
duration = 2,
|
||||
videoWidth = 640,
|
||||
videoHeight = 360,
|
||||
}: {
|
||||
duration?: number;
|
||||
videoWidth?: number;
|
||||
videoHeight?: number;
|
||||
} = {}): VideoElementHandle & {
|
||||
seekCalls: number[];
|
||||
drawCalls: number;
|
||||
disposed: boolean;
|
||||
} {
|
||||
const handle = {
|
||||
seekCalls: [] as number[],
|
||||
drawCalls: 0,
|
||||
disposed: false,
|
||||
duration,
|
||||
videoWidth,
|
||||
videoHeight,
|
||||
async seek(time: number) {
|
||||
handle.seekCalls.push(time);
|
||||
},
|
||||
drawTo() {
|
||||
handle.drawCalls += 1;
|
||||
},
|
||||
dispose() {
|
||||
handle.disposed = true;
|
||||
},
|
||||
};
|
||||
return handle;
|
||||
}
|
||||
|
||||
describe("clampFallbackFps", () => {
|
||||
test("returns default for undefined/NaN/zero/negative", () => {
|
||||
expect(clampFallbackFps(undefined)).toBe(30);
|
||||
expect(clampFallbackFps(Number.NaN)).toBe(30);
|
||||
expect(clampFallbackFps(0)).toBe(30);
|
||||
expect(clampFallbackFps(-5)).toBe(30);
|
||||
});
|
||||
|
||||
test("rounds and clamps within [1, 60]", () => {
|
||||
expect(clampFallbackFps(24)).toBe(24);
|
||||
expect(clampFallbackFps(29.4)).toBe(29);
|
||||
expect(clampFallbackFps(120)).toBe(60);
|
||||
expect(clampFallbackFps(0.4)).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isWebCodecsAvailable", () => {
|
||||
const root = globalThis as { VideoDecoder?: unknown };
|
||||
let originalVideoDecoder: unknown;
|
||||
|
||||
beforeEach(() => {
|
||||
originalVideoDecoder = root.VideoDecoder;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (originalVideoDecoder === undefined) {
|
||||
delete root.VideoDecoder;
|
||||
} else {
|
||||
root.VideoDecoder = originalVideoDecoder;
|
||||
}
|
||||
});
|
||||
|
||||
test("false when VideoDecoder is undefined", () => {
|
||||
delete root.VideoDecoder;
|
||||
expect(isWebCodecsAvailable()).toBe(false);
|
||||
});
|
||||
|
||||
test("true when VideoDecoder is defined", () => {
|
||||
root.VideoDecoder = class {};
|
||||
expect(isWebCodecsAvailable()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("HTMLVideoElementSink.canvases", () => {
|
||||
let dom: ReturnType<typeof installFakeDom>;
|
||||
|
||||
beforeEach(() => {
|
||||
dom = installFakeDom();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
dom.restore();
|
||||
});
|
||||
|
||||
test("yields frames stepping by 1/fps from startTimestamp", async () => {
|
||||
const handle = createFakeHandle({ duration: 0.25 });
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 10 }); // step = 0.1s
|
||||
|
||||
const frames: Array<{ timestamp: number; duration: number }> = [];
|
||||
for await (const frame of sink.canvases(0)) {
|
||||
frames.push({
|
||||
timestamp: frame.timestamp,
|
||||
duration: frame.duration,
|
||||
});
|
||||
}
|
||||
|
||||
expect(frames).toHaveLength(3);
|
||||
expect(frames[0].timestamp).toBeCloseTo(0, 5);
|
||||
expect(frames[1].timestamp).toBeCloseTo(0.1, 5);
|
||||
expect(frames[2].timestamp).toBeCloseTo(0.2, 5);
|
||||
expect(frames[0].duration).toBeCloseTo(0.1, 5);
|
||||
expect(handle.seekCalls).toEqual([0, 0.1, 0.2]);
|
||||
expect(handle.drawCalls).toBe(3);
|
||||
});
|
||||
|
||||
test("respects endTimestamp", async () => {
|
||||
const handle = createFakeHandle({ duration: 5 });
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 10 });
|
||||
|
||||
const frames: number[] = [];
|
||||
for await (const frame of sink.canvases(0, 0.2)) {
|
||||
frames.push(frame.timestamp);
|
||||
}
|
||||
|
||||
expect(frames).toHaveLength(2);
|
||||
expect(frames[0]).toBeCloseTo(0, 5);
|
||||
expect(frames[1]).toBeCloseTo(0.1, 5);
|
||||
});
|
||||
|
||||
test("dispose stops the iterator early", async () => {
|
||||
const handle = createFakeHandle({ duration: 10 });
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 10 });
|
||||
|
||||
const collected: number[] = [];
|
||||
for await (const frame of sink.canvases(0)) {
|
||||
collected.push(frame.timestamp);
|
||||
if (collected.length === 2) sink.dispose();
|
||||
}
|
||||
|
||||
expect(collected).toHaveLength(2);
|
||||
expect(handle.disposed).toBe(true);
|
||||
});
|
||||
|
||||
test("returns no frames when start >= duration", async () => {
|
||||
const handle = createFakeHandle({ duration: 1 });
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 30 });
|
||||
|
||||
const frames: number[] = [];
|
||||
for await (const frame of sink.canvases(2)) {
|
||||
frames.push(frame.timestamp);
|
||||
}
|
||||
expect(frames).toHaveLength(0);
|
||||
expect(handle.seekCalls).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("aborts cleanly when seek throws", async () => {
|
||||
const handle = createFakeHandle({ duration: 1 });
|
||||
handle.seek = async () => {
|
||||
throw new Error("boom");
|
||||
};
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 30 });
|
||||
|
||||
const frames: number[] = [];
|
||||
for await (const frame of sink.canvases(0)) {
|
||||
frames.push(frame.timestamp);
|
||||
}
|
||||
expect(frames).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("clamps invalid fps passed to constructor to the default", async () => {
|
||||
const handle = createFakeHandle({ duration: 0.05 });
|
||||
// Passing 0 (negative, or NaN) used to leak through to 1/fps and produce
|
||||
// invalid frame stepping. The constructor must reject these and fall back
|
||||
// to the default FPS.
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 0 });
|
||||
const frames = [];
|
||||
for await (const frame of sink.canvases(0)) {
|
||||
frames.push(frame);
|
||||
}
|
||||
expect(frames).toHaveLength(2);
|
||||
expect(frames[0].duration).toBeCloseTo(1 / 30, 5);
|
||||
});
|
||||
|
||||
test("frame canvas matches video dimensions", async () => {
|
||||
const handle = createFakeHandle({
|
||||
duration: 0.02,
|
||||
videoWidth: 1280,
|
||||
videoHeight: 720,
|
||||
});
|
||||
const sink = new HTMLVideoElementSink({ handle, fps: 30 });
|
||||
const frames = [];
|
||||
for await (const frame of sink.canvases(0)) {
|
||||
frames.push(frame);
|
||||
}
|
||||
expect(frames).toHaveLength(1);
|
||||
expect(frames[0].canvas.width).toBe(1280);
|
||||
expect(frames[0].canvas.height).toBe(720);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,249 @@
|
|||
import type { WrappedCanvas } from "mediabunny";
|
||||
|
||||
const DEFAULT_FALLBACK_FPS = 30;
|
||||
const MAX_FALLBACK_FPS = 60;
|
||||
const MIN_FALLBACK_FPS = 1;
|
||||
|
||||
export interface FrameSink {
|
||||
canvases(
|
||||
startTimestamp?: number,
|
||||
endTimestamp?: number,
|
||||
): AsyncGenerator<WrappedCanvas, void, unknown>;
|
||||
}
|
||||
|
||||
export interface VideoElementHandle {
|
||||
readonly duration: number;
|
||||
readonly videoWidth: number;
|
||||
readonly videoHeight: number;
|
||||
seek(time: number): Promise<void>;
|
||||
drawTo(target: {
|
||||
canvas: HTMLCanvasElement | OffscreenCanvas;
|
||||
ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
|
||||
}): void;
|
||||
dispose(): void;
|
||||
}
|
||||
|
||||
export function isWebCodecsAvailable(): boolean {
|
||||
return typeof (globalThis as { VideoDecoder?: unknown }).VideoDecoder !==
|
||||
"undefined";
|
||||
}
|
||||
|
||||
export function clampFallbackFps(requested: number | undefined): number {
|
||||
if (
|
||||
requested === undefined ||
|
||||
!Number.isFinite(requested) ||
|
||||
requested <= 0
|
||||
) {
|
||||
return DEFAULT_FALLBACK_FPS;
|
||||
}
|
||||
const rounded = Math.round(requested);
|
||||
if (rounded < MIN_FALLBACK_FPS) return MIN_FALLBACK_FPS;
|
||||
if (rounded > MAX_FALLBACK_FPS) return MAX_FALLBACK_FPS;
|
||||
return rounded;
|
||||
}
|
||||
|
||||
export async function createHTMLVideoElementHandle({
|
||||
file,
|
||||
}: {
|
||||
file: File;
|
||||
}): Promise<VideoElementHandle> {
|
||||
const objectUrl = URL.createObjectURL(file);
|
||||
const video = document.createElement("video");
|
||||
video.preload = "auto";
|
||||
video.muted = true;
|
||||
video.playsInline = true;
|
||||
video.crossOrigin = "anonymous";
|
||||
video.src = objectUrl;
|
||||
|
||||
try {
|
||||
await waitForLoadedMetadata({ video });
|
||||
} catch (error) {
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
throw error;
|
||||
}
|
||||
|
||||
let disposed = false;
|
||||
|
||||
return {
|
||||
get duration() {
|
||||
return video.duration;
|
||||
},
|
||||
get videoWidth() {
|
||||
return video.videoWidth;
|
||||
},
|
||||
get videoHeight() {
|
||||
return video.videoHeight;
|
||||
},
|
||||
async seek(time) {
|
||||
if (disposed) {
|
||||
throw new Error("VideoElementHandle disposed");
|
||||
}
|
||||
await seekVideoElement({ video, time });
|
||||
},
|
||||
drawTo({ canvas, ctx }) {
|
||||
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||||
},
|
||||
dispose() {
|
||||
if (disposed) return;
|
||||
disposed = true;
|
||||
try {
|
||||
video.pause();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
video.removeAttribute("src");
|
||||
try {
|
||||
video.load();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function waitForLoadedMetadata({
|
||||
video,
|
||||
}: {
|
||||
video: HTMLVideoElement;
|
||||
}): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const cleanup = () => {
|
||||
video.removeEventListener("loadedmetadata", onLoaded);
|
||||
video.removeEventListener("error", onError);
|
||||
};
|
||||
const onLoaded = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = () => {
|
||||
cleanup();
|
||||
const msg = video.error?.message ?? "unknown error";
|
||||
reject(new Error(`HTMLVideoElement load failed: ${msg}`));
|
||||
};
|
||||
video.addEventListener("loadedmetadata", onLoaded);
|
||||
video.addEventListener("error", onError);
|
||||
});
|
||||
}
|
||||
|
||||
function seekVideoElement({
|
||||
video,
|
||||
time,
|
||||
}: {
|
||||
video: HTMLVideoElement;
|
||||
time: number;
|
||||
}): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const cleanup = () => {
|
||||
video.removeEventListener("seeked", onSeeked);
|
||||
video.removeEventListener("error", onError);
|
||||
};
|
||||
const onSeeked = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = () => {
|
||||
cleanup();
|
||||
reject(new Error("HTMLVideoElement seek failed"));
|
||||
};
|
||||
video.addEventListener("seeked", onSeeked);
|
||||
video.addEventListener("error", onError);
|
||||
try {
|
||||
video.currentTime = time;
|
||||
} catch (err) {
|
||||
cleanup();
|
||||
reject(err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export class HTMLVideoElementSink implements FrameSink {
|
||||
private disposed = false;
|
||||
private readonly handle: VideoElementHandle;
|
||||
private readonly fps: number;
|
||||
|
||||
constructor({
|
||||
handle,
|
||||
fps,
|
||||
}: {
|
||||
handle: VideoElementHandle;
|
||||
fps?: number;
|
||||
}) {
|
||||
this.handle = handle;
|
||||
this.fps = clampFallbackFps(fps);
|
||||
}
|
||||
|
||||
static async create({
|
||||
file,
|
||||
fps,
|
||||
}: {
|
||||
file: File;
|
||||
fps?: number;
|
||||
}): Promise<HTMLVideoElementSink> {
|
||||
const handle = await createHTMLVideoElementHandle({ file });
|
||||
return new HTMLVideoElementSink({ handle, fps: clampFallbackFps(fps) });
|
||||
}
|
||||
|
||||
// eslint-disable-next-line opencut/prefer-object-params -- mirrors mediabunny's CanvasSink.canvases(start, end) so VideoCache can use either sink transparently
|
||||
async *canvases(
|
||||
startTimestamp = 0,
|
||||
endTimestamp?: number,
|
||||
): AsyncGenerator<WrappedCanvas, void, unknown> {
|
||||
const step = 1 / this.fps;
|
||||
const duration = this.handle.duration;
|
||||
const end =
|
||||
typeof endTimestamp === "number" && Number.isFinite(endTimestamp)
|
||||
? Math.min(endTimestamp, duration)
|
||||
: duration;
|
||||
let t = Math.max(0, startTimestamp);
|
||||
|
||||
while (t < end && !this.disposed) {
|
||||
try {
|
||||
await this.handle.seek(t);
|
||||
} catch (error) {
|
||||
console.warn("[HTMLVideoElementSink] seek failed:", error);
|
||||
return;
|
||||
}
|
||||
if (this.disposed) return;
|
||||
|
||||
const canvas = createFrameCanvas({
|
||||
width: this.handle.videoWidth,
|
||||
height: this.handle.videoHeight,
|
||||
});
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) {
|
||||
console.warn(
|
||||
"[HTMLVideoElementSink] 2d context unavailable, aborting iterator",
|
||||
);
|
||||
return;
|
||||
}
|
||||
this.handle.drawTo({ canvas, ctx });
|
||||
|
||||
yield {
|
||||
canvas,
|
||||
timestamp: t,
|
||||
duration: step,
|
||||
};
|
||||
t += step;
|
||||
}
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
if (this.disposed) return;
|
||||
this.disposed = true;
|
||||
this.handle.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
function createFrameCanvas({
|
||||
width,
|
||||
height,
|
||||
}: {
|
||||
width: number;
|
||||
height: number;
|
||||
}): HTMLCanvasElement {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = Math.max(1, Math.round(width));
|
||||
canvas.height = Math.max(1, Math.round(height));
|
||||
return canvas;
|
||||
}
|
||||
|
|
@ -5,10 +5,15 @@ import {
|
|||
CanvasSink,
|
||||
type WrappedCanvas,
|
||||
} from "mediabunny";
|
||||
import {
|
||||
type FrameSink,
|
||||
HTMLVideoElementSink,
|
||||
isWebCodecsAvailable,
|
||||
} from "./html-video-element-sink";
|
||||
|
||||
interface VideoSinkData {
|
||||
input: Input;
|
||||
sink: CanvasSink;
|
||||
sink: FrameSink;
|
||||
dispose: () => void;
|
||||
iterator: AsyncGenerator<WrappedCanvas, void, unknown> | null;
|
||||
currentFrame: WrappedCanvas | null;
|
||||
nextFrame: WrappedCanvas | null;
|
||||
|
|
@ -275,7 +280,15 @@ export class VideoCache {
|
|||
|
||||
const canDecode = await videoTrack.canDecode();
|
||||
if (!canDecode) {
|
||||
throw new Error("Video codec not supported for decoding");
|
||||
if (isWebCodecsAvailable()) {
|
||||
throw new Error("Video codec not supported for decoding");
|
||||
}
|
||||
// WebCodecs API absent (e.g. browsers without VideoDecoder support).
|
||||
// Fall back to an HTMLVideoElement-backed sink which uses the
|
||||
// system's native video decoders via the <video> tag.
|
||||
input.dispose();
|
||||
await this.initializeHTMLVideoElementSink({ mediaId, file });
|
||||
return;
|
||||
}
|
||||
|
||||
const sink = new CanvasSink(videoTrack, {
|
||||
|
|
@ -284,8 +297,8 @@ export class VideoCache {
|
|||
});
|
||||
|
||||
this.sinks.set(mediaId, {
|
||||
input,
|
||||
sink,
|
||||
dispose: () => input.dispose(),
|
||||
iterator: null,
|
||||
currentFrame: null,
|
||||
nextFrame: null,
|
||||
|
|
@ -300,6 +313,29 @@ export class VideoCache {
|
|||
}
|
||||
}
|
||||
|
||||
private async initializeHTMLVideoElementSink({
|
||||
mediaId,
|
||||
file,
|
||||
}: {
|
||||
mediaId: string;
|
||||
file: File;
|
||||
}): Promise<void> {
|
||||
const fallbackSink = await HTMLVideoElementSink.create({ file });
|
||||
this.sinks.set(mediaId, {
|
||||
sink: fallbackSink,
|
||||
dispose: () => fallbackSink.dispose(),
|
||||
iterator: null,
|
||||
currentFrame: null,
|
||||
nextFrame: null,
|
||||
lastTime: -1,
|
||||
prefetching: false,
|
||||
prefetchPromise: null,
|
||||
});
|
||||
console.warn(
|
||||
"[VideoCache] WebCodecs unavailable; using HTMLVideoElement fallback for video preview.",
|
||||
);
|
||||
}
|
||||
|
||||
clearVideo({ mediaId }: { mediaId: string }): void {
|
||||
const sinkData = this.sinks.get(mediaId);
|
||||
if (sinkData) {
|
||||
|
|
@ -307,7 +343,7 @@ export class VideoCache {
|
|||
void sinkData.iterator.return();
|
||||
}
|
||||
|
||||
sinkData.input.dispose();
|
||||
sinkData.dispose();
|
||||
this.sinks.delete(mediaId);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue