RomanNum3ral
/
OpenCut
mirror of https://github.com/OpenCut-app/OpenCut.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for basic SRT subtitle import to Captions panel (#756) 

* Add SRT subtitle import to the captions panel

[Modified from 0bd641b4102c827ef8d218ada099229f73396077, 2026-03-28 08:46:23 -0400]

Parse .srt files into chunks and insert them as text elements on a new text track using the existing caption styling defaults. Displays a warning when malformed cues are skipped during import.

* Batch caption text insertion

[Modified from eece3bbc5f0578a24a4f90f0ab87fe5ca37e94e9, 2026-03-28 09:13:18 -0400]

Batch caption insertion in a BatchCommand so that undo will batch undo all subtitle insertion from either srt file or ai transcriptions.

* Move subtitle parsing into a shared interface

[Modified from e84404b7857f8c611eda7a65b5d61ee6c9823d75, 2026-03-28 09:52:31 -0400]

Extracted the srt specific extraction behind a generic shared interface to allow easier adding of other subtitle formats in the future.

* Apply better default styling for subtitles.

[Modified from b5deec4285f99873ba5bf57971f7bcab26e8c743, 2026-03-31 11:49:34 -0400]

Shrink font size, aim to layout 5% above bottom, keep subtitle width 80% of video, wrap text on new lines if too long.

* Update Captions Panel UI

Split captions panel into Generate and Import tabs as they are two different approaches for adding captions.

* Add biome formatting fixes

* Add .ass subtitle import support

Extend subtitle import to accept .ass files alongside .srt, map supported .ass styles onto imported text elements, and display warnings when unsupported .ass features are skipped.

* Change “Import subtitles” button style

* refactor: review fixes

* fix: convert subtitle times from seconds to ticks

* refactor: replace captions tab switcher with import action button

---------

Co-authored-by: Maze Winther <mazewinther@gmail.com>
This commit is contained in:
kcfancher 2026-04-13 21:19:12 -04:00 committed by GitHub
parent d85566ab8d
commit e3d4f3fa92
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 1341 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

461
apps/web/src/components/editor/panels/assets/views/captions.tsx
View File

@ -1,179 +1,282 @@
import { Button } from "@/components/ui/button";
import { PanelView } from "@/components/editor/panels/assets/views/base-panel";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { useState, useRef } from "react";
import { extractTimelineAudio } from "@/lib/media/mediabunny";
import { useEditor } from "@/hooks/use-editor";
import {
DEFAULT_TRANSCRIPTION_SAMPLE_RATE,
TRANSCRIPTION_LANGUAGES,
} from "@/constants/transcription-constants";
import type {
TranscriptionLanguage,
TranscriptionProgress,
} from "@/lib/transcription/types";
import { transcriptionService } from "@/services/transcription/service";
import { decodeAudioToFloat32 } from "@/lib/media/audio";
import { buildCaptionChunks } from "@/lib/transcription/caption";
import { Spinner } from "@/components/ui/spinner";
import {
Section,
SectionContent,
SectionField,
SectionFields,
} from "@/components/section";
import { DEFAULTS } from "@/lib/timeline/defaults";
import {
AddTrackCommand,
BatchCommand,
InsertElementCommand,
} from "@/lib/commands";
export function Captions() {
const [selectedLanguage, setSelectedLanguage] =
useState<TranscriptionLanguage>("auto");
const [isProcessing, setIsProcessing] = useState(false);
const [processingStep, setProcessingStep] = useState("");
const [error, setError] = useState<string | null>(null);
const containerRef = useRef<HTMLDivElement>(null);
const editor = useEditor();
const handleProgress = (progress: TranscriptionProgress) => {
if (progress.status === "loading-model") {
setProcessingStep(`Loading model ${Math.round(progress.progress)}%`);
} else if (progress.status === "transcribing") {
setProcessingStep("Transcribing...");
}
};
const handleGenerateTranscript = async () => {
try {
setIsProcessing(true);
setError(null);
setProcessingStep("Extracting audio...");
const audioBlob = await extractTimelineAudio({
tracks: editor.scenes.getActiveScene().tracks,
mediaAssets: editor.media.getAssets(),
totalDuration: editor.timeline.getTotalDuration(),
});
setProcessingStep("Preparing audio...");
const { samples } = await decodeAudioToFloat32({
audioBlob,
sampleRate: DEFAULT_TRANSCRIPTION_SAMPLE_RATE,
});
const result = await transcriptionService.transcribe({
audioData: samples,
language: selectedLanguage === "auto" ? undefined : selectedLanguage,
onProgress: handleProgress,
});
setProcessingStep("Generating captions...");
const captionChunks = buildCaptionChunks({ segments: result.segments });
const addTrackCommand = new AddTrackCommand("text", 0);
const insertCommands = captionChunks.map(
(caption, i) =>
new InsertElementCommand({
placement: {
mode: "explicit",
trackId: addTrackCommand.getTrackId(),
},
element: {
...DEFAULTS.text.element,
name: `Caption ${i + 1}`,
content: caption.text,
duration: caption.duration,
startTime: caption.startTime,
fontSize: 65,
fontWeight: "bold",
},
}),
);
editor.command.execute({
command: new BatchCommand([addTrackCommand, ...insertCommands]),
});
} catch (error) {
console.error("Transcription failed:", error);
setError(
error instanceof Error ? error.message : "An unexpected error occurred",
);
} finally {
setIsProcessing(false);
setProcessingStep("");
}
};
const handleLanguageChange = ({ value }: { value: string }) => {
if (value === "auto") {
setSelectedLanguage("auto");
return;
}
const matchedLanguage = TRANSCRIPTION_LANGUAGES.find(
(language) => language.code === value,
);
if (!matchedLanguage) return;
setSelectedLanguage(matchedLanguage.code);
};
return (
<PanelView
title="Captions"
contentClassName="px-0 flex flex-col h-full"
ref={containerRef}
>
<Section showTopBorder={false} showBottomBorder={false} className="flex-1">
<SectionContent className="flex flex-col gap-4 h-full pt-1">
<SectionFields>
<SectionField label="Language">
<Select
value={selectedLanguage}
onValueChange={(value) => handleLanguageChange({ value })}
>
<SelectTrigger>
<SelectValue placeholder="Select a language" />
</SelectTrigger>
<SelectContent>
<SelectItem value="auto">Auto detect</SelectItem>
{TRANSCRIPTION_LANGUAGES.map((language) => (
<SelectItem key={language.code} value={language.code}>
{language.name}
</SelectItem>
))}
</SelectContent>
</Select>
</SectionField>
</SectionFields>
{error && (
<div className="bg-destructive/10 border-destructive/20 rounded-md border p-3">
<p className="text-destructive text-sm">{error}</p>
</div>
)}
</SectionContent>
</Section>
<Section showBottomBorder={false} showTopBorder={false}>
<SectionContent>
<Button
className="w-full"
onClick={handleGenerateTranscript}
disabled={isProcessing}
>
{isProcessing && <Spinner className="mr-1" />}
{isProcessing ? processingStep : "Generate transcript"}
</Button>
</SectionContent>
</Section>
</PanelView>
);
}
import { Button } from "@/components/ui/button";
import { PanelView } from "@/components/editor/panels/assets/views/base-panel";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { useReducer, useRef, useState } from "react";
import { extractTimelineAudio } from "@/lib/media/mediabunny";
import { useEditor } from "@/hooks/use-editor";
import {
DEFAULT_TRANSCRIPTION_SAMPLE_RATE,
TRANSCRIPTION_LANGUAGES,
} from "@/constants/transcription-constants";
import type {
CaptionChunk,
TranscriptionLanguage,
TranscriptionProgress,
} from "@/lib/transcription/types";
import { transcriptionService } from "@/services/transcription/service";
import { decodeAudioToFloat32 } from "@/lib/media/audio";
import { buildCaptionChunks } from "@/lib/transcription/caption";
import { insertCaptionChunksAsTextTrack } from "@/lib/subtitles/insert";
import { parseSubtitleFile } from "@/lib/subtitles/parse";
import { Spinner } from "@/components/ui/spinner";
import {
Section,
SectionContent,
SectionField,
SectionFields,
} from "@/components/section";
import { CloudUploadIcon } from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
type ProcessingState =
| { status: "idle"; error: string | null; warnings: string[] }
| { status: "processing"; step: string };
type ProcessingAction =
| { type: "start"; step: string }
| { type: "update_step"; step: string }
| { type: "succeed"; warnings: string[] }
| { type: "fail"; error: string };
const IDLE_STATE: ProcessingState = { status: "idle", error: null, warnings: [] };
function processingReducer(
state: ProcessingState,
action: ProcessingAction,
): ProcessingState {
switch (action.type) {
case "start":
return { status: "processing", step: action.step };
case "update_step":
if (state.status !== "processing") return state;
return { status: "processing", step: action.step };
case "succeed":
return { status: "idle", error: null, warnings: action.warnings };
case "fail":
return { status: "idle", error: action.error, warnings: [] };
}
}
export function Captions() {
const [selectedLanguage, setSelectedLanguage] =
useState<TranscriptionLanguage>("auto");
const [processing, dispatch] = useReducer(processingReducer, IDLE_STATE);
const containerRef = useRef<HTMLDivElement>(null);
const fileInputRef = useRef<HTMLInputElement>(null);
const editor = useEditor();
const isProcessing = processing.status === "processing";
const handleProgress = (progress: TranscriptionProgress) => {
if (progress.status === "loading-model") {
dispatch({
type: "update_step",
step: `Loading model ${Math.round(progress.progress)}%`,
});
} else if (progress.status === "transcribing") {
dispatch({ type: "update_step", step: "Transcribing..." });
}
};
const insertCaptions = ({ captions }: { captions: CaptionChunk[] }): boolean => {
const trackId = insertCaptionChunksAsTextTrack({ editor, captions });
return trackId !== null;
};
const handleGenerateTranscript = async () => {
dispatch({ type: "start", step: "Extracting audio..." });
try {
const audioBlob = await extractTimelineAudio({
tracks: editor.scenes.getActiveScene().tracks,
mediaAssets: editor.media.getAssets(),
totalDuration: editor.timeline.getTotalDuration(),
});
dispatch({ type: "update_step", step: "Preparing audio..." });
const { samples } = await decodeAudioToFloat32({
audioBlob,
sampleRate: DEFAULT_TRANSCRIPTION_SAMPLE_RATE,
});
const result = await transcriptionService.transcribe({
audioData: samples,
language: selectedLanguage === "auto" ? undefined : selectedLanguage,
onProgress: handleProgress,
});
dispatch({ type: "update_step", step: "Generating captions..." });
const captionChunks = buildCaptionChunks({ segments: result.segments });
if (!insertCaptions({ captions: captionChunks })) {
dispatch({ type: "fail", error: "No captions were generated" });
return;
}
dispatch({ type: "succeed", warnings: [] });
} catch (error) {
console.error("Transcription failed:", error);
dispatch({
type: "fail",
error: error instanceof Error ? error.message : "An unexpected error occurred",
});
}
};
const handleImportClick = () => {
fileInputRef.current?.click();
};
const handleImportFile = async ({ file }: { file: File }) => {
dispatch({ type: "start", step: "Reading subtitle file..." });
try {
const input = await file.text();
const result = parseSubtitleFile({
fileName: file.name,
input,
});
if (result.captions.length === 0) {
dispatch({
type: "fail",
error: "No valid subtitle cues were found in the subtitle file",
});
return;
}
dispatch({ type: "update_step", step: "Importing subtitles..." });
if (!insertCaptions({ captions: result.captions })) {
dispatch({ type: "fail", error: "No captions were generated" });
return;
}
const nextWarnings = [...result.warnings];
if (result.skippedCueCount > 0) {
nextWarnings.unshift(
`Imported ${result.captions.length} subtitle cue(s) and skipped ${result.skippedCueCount} malformed cue(s).`,
);
}
dispatch({ type: "succeed", warnings: nextWarnings });
} catch (error) {
console.error("Subtitle import failed:", error);
dispatch({
type: "fail",
error: error instanceof Error ? error.message : "An unexpected error occurred",
});
}
};
const handleFileChange = async ({
event,
}: {
event: React.ChangeEvent<HTMLInputElement>;
}) => {
const file = event.target.files?.[0];
if (event.target) {
event.target.value = "";
}
if (!file) return;
await handleImportFile({ file });
};
const handleLanguageChange = ({ value }: { value: string }) => {
if (value === "auto") {
setSelectedLanguage("auto");
return;
}
const matchedLanguage = TRANSCRIPTION_LANGUAGES.find(
(language) => language.code === value,
);
if (!matchedLanguage) return;
setSelectedLanguage(matchedLanguage.code);
};
const error = processing.status === "idle" ? processing.error : null;
const warnings = processing.status === "idle" ? processing.warnings : [];
return (
<PanelView
title="Captions"
contentClassName="px-0 flex flex-col h-full"
actions={
<Button
type="button"
variant="outline"
size="sm"
onClick={handleImportClick}
disabled={isProcessing}
className="items-center justify-center gap-1.5"
>
<HugeiconsIcon icon={CloudUploadIcon} />
Import
</Button>
}
ref={containerRef}
>
<input
ref={fileInputRef}
type="file"
accept=".srt,.ass"
className="hidden"
onChange={(event) => void handleFileChange({ event })}
/>
<Section showTopBorder={false} showBottomBorder={false} className="flex-1">
<SectionContent className="flex flex-col gap-4 h-full pt-1">
<SectionFields>
<SectionField label="Language">
<Select
value={selectedLanguage}
onValueChange={(value) => handleLanguageChange({ value })}
>
<SelectTrigger>
<SelectValue placeholder="Select a language" />
</SelectTrigger>
<SelectContent>
<SelectItem value="auto">Auto detect</SelectItem>
{TRANSCRIPTION_LANGUAGES.map((language) => (
<SelectItem key={language.code} value={language.code}>
{language.name}
</SelectItem>
))}
</SelectContent>
</Select>
</SectionField>
</SectionFields>
<Button
type="button"
className="mt-auto w-full"
onClick={handleGenerateTranscript}
disabled={isProcessing}
>
{isProcessing && <Spinner className="mr-1" />}
{isProcessing ? processing.step : "Generate transcript"}
</Button>
{error && (
<div className="bg-destructive/10 border-destructive/20 rounded-md border p-3">
<p className="text-destructive text-sm">{error}</p>
</div>
)}
{warnings.length > 0 && (
<div className="rounded-md border border-amber-500/20 bg-amber-500/10 p-3">
<ul className="space-y-1 text-sm text-amber-700">
{warnings.map((warning) => (
<li key={warning}>{warning}</li>
))}
</ul>
</div>
)}
</SectionContent>
</Section>
</PanelView>
);
}

514
apps/web/src/lib/subtitles/ass.ts Normal file
View File

@ -0,0 +1,514 @@
import type {
ParseSubtitleResult,
SubtitleCue,
SubtitleStyleOverrides,
} from "./types";
const ASS_DEFAULT_PLAY_RES_X = 384;
const ASS_DEFAULT_PLAY_RES_Y = 288;
const STYLE_SECTION_NAMES = new Set(["v4 styles", "v4+ styles"]);
const ALIGNMENT_MAP: Record<
number,
{
textAlign: NonNullable<SubtitleStyleOverrides["textAlign"]>;
verticalAlign: NonNullable<
NonNullable<SubtitleStyleOverrides["placement"]>["verticalAlign"]
>;
}
> = {
1: { textAlign: "left", verticalAlign: "bottom" },
2: { textAlign: "center", verticalAlign: "bottom" },
3: { textAlign: "right", verticalAlign: "bottom" },
4: { textAlign: "left", verticalAlign: "middle" },
5: { textAlign: "center", verticalAlign: "middle" },
6: { textAlign: "right", verticalAlign: "middle" },
7: { textAlign: "left", verticalAlign: "top" },
8: { textAlign: "center", verticalAlign: "top" },
9: { textAlign: "right", verticalAlign: "top" },
};
interface AssScriptInfo {
playResX: number;
playResY: number;
}
interface AssStyleRecord {
name: string;
fontname?: string;
fontsize?: string;
primarycolour?: string;
secondarycolour?: string;
outlinecolour?: string;
backcolour?: string;
bold?: string;
italic?: string;
underline?: string;
strikeout?: string;
scalex?: string;
scaley?: string;
spacing?: string;
angle?: string;
borderstyle?: string;
outline?: string;
shadow?: string;
alignment?: string;
marginl?: string;
marginr?: string;
marginv?: string;
}
export function parseAss({ input }: { input: string }): ParseSubtitleResult {
const normalized = input.replace(/\r\n?/g, "\n").trim();
if (!normalized) {
return {
captions: [],
skippedCueCount: 0,
warnings: [],
};
}
const scriptInfo: AssScriptInfo = {
playResX: ASS_DEFAULT_PLAY_RES_X,
playResY: ASS_DEFAULT_PLAY_RES_Y,
};
const warnings = new Set<string>();
const styles = new Map<string, SubtitleStyleOverrides>();
const captions: SubtitleCue[] = [];
let currentSection = "";
let styleFormat: string[] | null = null;
let eventFormat: string[] | null = null;
let skippedCueCount = 0;
let strippedInlineTagCueCount = 0;
let skippedNonDialogueEventCount = 0;
let ignoredEffectCount = 0;
let missingStyleCueCount = 0;
let usesHeavilyUnsupportedStyles = false;
for (const rawLine of normalized.split("\n")) {
const line = rawLine.trim();
if (!line || line.startsWith(";")) {
continue;
}
const sectionMatch = line.match(/^\[(.+)\]$/);
if (sectionMatch) {
currentSection = sectionMatch[1].trim().toLowerCase();
continue;
}
if (currentSection === "script info") {
const [rawKey, ...rest] = line.split(":");
if (!rawKey || rest.length === 0) continue;
const key = rawKey.trim().toLowerCase();
const value = rest.join(":").trim();
const parsedValue = parseFloat(value);
if (!Number.isFinite(parsedValue) || parsedValue <= 0) {
continue;
}
if (key === "playresx") {
scriptInfo.playResX = parsedValue;
}
if (key === "playresy") {
scriptInfo.playResY = parsedValue;
}
continue;
}
if (STYLE_SECTION_NAMES.has(currentSection)) {
if (line.startsWith("Format:")) {
styleFormat = parseAssFormat({ line });
continue;
}
if (line.startsWith("Style:") && styleFormat) {
const values = splitAssFields({
value: line.slice("Style:".length).trim(),
expectedFieldCount: styleFormat.length,
});
if (values.length !== styleFormat.length) {
continue;
}
const record = mapFieldsToRecord<AssStyleRecord>({
fields: styleFormat,
values,
});
if (!record.name) {
continue;
}
const mappedStyle = mapAssStyleToSubtitleStyle({
style: record,
scriptInfo,
});
usesHeavilyUnsupportedStyles ||= mappedStyle.hasUnsupportedFeatures;
styles.set(record.name.toLowerCase(), mappedStyle.style);
}
continue;
}
if (currentSection !== "events") {
continue;
}
if (line.startsWith("Format:")) {
eventFormat = parseAssFormat({ line });
continue;
}
if (!eventFormat) {
continue;
}
if (line.startsWith("Dialogue:")) {
const values = splitAssFields({
value: line.slice("Dialogue:".length).trim(),
expectedFieldCount: eventFormat.length,
});
if (values.length !== eventFormat.length) {
skippedCueCount += 1;
continue;
}
const record = mapFieldsToRecord<Record<string, string>>({
fields: eventFormat,
values,
});
const startTime = parseAssTimestamp({ input: record.start });
const endTime = parseAssTimestamp({ input: record.end });
const duration = endTime - startTime;
if (
!Number.isFinite(startTime) ||
!Number.isFinite(endTime) ||
duration <= 0
) {
skippedCueCount += 1;
continue;
}
const strippedText = stripAssText({
input: record.text ?? "",
});
strippedInlineTagCueCount += strippedText.hadInlineTags ? 1 : 0;
if (!strippedText.text) {
skippedCueCount += 1;
continue;
}
if (record.effect?.trim()) {
ignoredEffectCount += 1;
}
const referencedStyleName = record.style?.trim().toLowerCase();
const referencedStyle = referencedStyleName
? styles.get(referencedStyleName)
: undefined;
if (referencedStyleName && !referencedStyle) {
missingStyleCueCount += 1;
}
const style = referencedStyle ?? styles.get("default");
captions.push({
text: strippedText.text,
startTime,
duration,
style,
});
continue;
}
if (line.includes(":")) {
skippedNonDialogueEventCount += 1;
}
}
if (strippedInlineTagCueCount > 0) {
warnings.add(
`Stripped unsupported ASS inline override tags from ${strippedInlineTagCueCount} subtitle cue(s).`,
);
}
if (ignoredEffectCount > 0) {
warnings.add(
`Ignored ASS event effects in ${ignoredEffectCount} subtitle cue(s).`,
);
}
if (missingStyleCueCount > 0) {
warnings.add(
`Fell back to default subtitle styling for ${missingStyleCueCount} cue(s) that referenced missing ASS styles.`,
);
}
if (skippedNonDialogueEventCount > 0) {
warnings.add(
`Ignored ${skippedNonDialogueEventCount} non-dialogue ASS event(s).`,
);
}
if (usesHeavilyUnsupportedStyles) {
warnings.add(
"Ignored unsupported ASS style features such as outline, shadow, rotation, or scaling.",
);
}
return {
captions,
skippedCueCount,
warnings: [...warnings],
};
}
function parseAssFormat({ line }: { line: string }): string[] {
return line
.slice(line.indexOf(":") + 1)
.split(",")
.map((field) => field.trim().toLowerCase());
}
function splitAssFields({
value,
expectedFieldCount,
}: {
value: string;
expectedFieldCount: number;
}): string[] {
if (expectedFieldCount <= 1) {
return [value];
}
const result: string[] = [];
let current = "";
for (const character of value) {
if (character === "," && result.length < expectedFieldCount - 1) {
result.push(current.trim());
current = "";
continue;
}
current += character;
}
result.push(current.trim());
return result;
}
function mapFieldsToRecord<T extends object>({
fields,
values,
}: {
fields: string[];
values: string[];
}): T {
const record = {} as Record<string, string | undefined>;
for (let index = 0; index < fields.length; index++) {
record[fields[index]] = values[index];
}
return record as T;
}
function parseAssTimestamp({ input }: { input: string | undefined }): number {
if (!input) {
return Number.NaN;
}
const match = input.trim().match(/^(\d+):(\d{2}):(\d{2})[.](\d{1,2}|\d{3})$/);
if (!match) {
return Number.NaN;
}
const [, hours, minutes, seconds, fraction] = match;
const parsedHours = Number.parseInt(hours, 10);
const parsedMinutes = Number.parseInt(minutes, 10);
const parsedSeconds = Number.parseInt(seconds, 10);
const parsedMilliseconds = Number.parseInt(fraction.padEnd(3, "0"), 10);
return (
parsedHours * 3600 +
parsedMinutes * 60 +
parsedSeconds +
parsedMilliseconds / 1000
);
}
function stripAssText({ input }: { input: string }): {
text: string;
hadInlineTags: boolean;
} {
const hadInlineTags = /\{[^}]*\}/.test(input);
const text = input
.replace(/\{[^}]*\}/g, "")
.replace(/\\N/gi, "\n")
.replace(/\\h/g, " ")
.replace(/\\n/g, "\n")
.trim();
return {
text,
hadInlineTags,
};
}
function mapAssStyleToSubtitleStyle({
style,
scriptInfo,
}: {
style: AssStyleRecord;
scriptInfo: AssScriptInfo;
}): {
style: SubtitleStyleOverrides;
hasUnsupportedFeatures: boolean;
} {
const fontSize = parseFloat(style.fontsize ?? "");
const primaryColor = parseAssColor({ input: style.primarycolour });
const backColor = parseAssColor({ input: style.backcolour });
const bold = parseAssBoolean({ input: style.bold });
const italic = parseAssBoolean({ input: style.italic });
const underline = parseAssBoolean({ input: style.underline });
const strikeOut = parseAssBoolean({ input: style.strikeout });
const borderStyle = parseFloat(style.borderstyle ?? "");
const spacing = parseFloat(style.spacing ?? "");
const alignment = parseFloat(style.alignment ?? "");
const marginLeft = parseFloat(style.marginl ?? "");
const marginRight = parseFloat(style.marginr ?? "");
const marginVertical = parseFloat(style.marginv ?? "");
// Store as a ratio of playResY so the builder can convert to app units
// without the parser needing to know the app's coordinate system.
const fontSizeRatioOfPlayHeight = Number.isFinite(fontSize)
? Math.round((fontSize / scriptInfo.playResY) * 1000) / 1000
: null;
const mappedAlignment =
ALIGNMENT_MAP[Math.round(alignment)] ?? ALIGNMENT_MAP[2];
const placement =
Number.isFinite(marginLeft) ||
Number.isFinite(marginRight) ||
Number.isFinite(marginVertical) ||
mappedAlignment.verticalAlign !== "bottom"
? {
verticalAlign: mappedAlignment.verticalAlign,
marginLeftRatio: Number.isFinite(marginLeft)
? marginLeft / scriptInfo.playResX
: undefined,
marginRightRatio: Number.isFinite(marginRight)
? marginRight / scriptInfo.playResX
: undefined,
marginVerticalRatio: Number.isFinite(marginVertical)
? marginVertical / scriptInfo.playResY
: undefined,
}
: undefined;
const styleOverrides: SubtitleStyleOverrides = {
...(style.fontname ? { fontFamily: style.fontname.trim() } : {}),
...(fontSizeRatioOfPlayHeight !== null && fontSizeRatioOfPlayHeight > 0
? { fontSizeRatioOfPlayHeight }
: {}),
...(primaryColor?.cssColor ? { color: primaryColor.cssColor } : {}),
...(bold !== null ? { fontWeight: bold ? "bold" : "normal" } : {}),
...(italic !== null ? { fontStyle: italic ? "italic" : "normal" } : {}),
...(underline || strikeOut
? {
textDecoration: underline
? "underline"
: strikeOut
? "line-through"
: "none",
}
: {}),
...(Number.isFinite(spacing) ? { letterSpacing: spacing } : {}),
textAlign: mappedAlignment.textAlign,
...(placement ? { placement } : {}),
...(backColor?.cssColor && Math.round(borderStyle) === 3
? {
background: {
enabled: backColor.alpha > 0,
color: backColor.alpha > 0 ? backColor.cssColor : "transparent",
},
}
: {}),
};
const hasUnsupportedFeatures =
Math.round(borderStyle) !== 1 && Math.round(borderStyle) !== 3
? true
: (parseFloat(style.outline ?? "") || 0) > 0 ||
(parseFloat(style.shadow ?? "") || 0) > 0 ||
(parseFloat(style.angle ?? "") || 0) !== 0 ||
(Number.isFinite(parseFloat(style.scalex ?? "")) &&
parseFloat(style.scalex ?? "") !== 100) ||
(Number.isFinite(parseFloat(style.scaley ?? "")) &&
parseFloat(style.scaley ?? "") !== 100) ||
Boolean(underline && strikeOut);
return {
style: styleOverrides,
hasUnsupportedFeatures,
};
}
function parseAssBoolean({
input,
}: {
input: string | undefined;
}): boolean | null {
if (input === undefined) {
return null;
}
const trimmed = input.trim();
if (!trimmed) {
return null;
}
const parsed = Number.parseInt(trimmed, 10);
if (!Number.isFinite(parsed)) {
return null;
}
return parsed !== 0;
}
function parseAssColor({
input,
}: {
input: string | undefined;
}): { cssColor: string; alpha: number } | null {
if (!input) {
return null;
}
const normalized = input.trim().replace(/^&?H/i, "").padStart(8, "0");
if (!/^[0-9a-fA-F]{8}$/.test(normalized)) {
return null;
}
const alphaHex = normalized.slice(0, 2);
const blueHex = normalized.slice(2, 4);
const greenHex = normalized.slice(4, 6);
const redHex = normalized.slice(6, 8);
const red = Number.parseInt(redHex, 16);
const green = Number.parseInt(greenHex, 16);
const blue = Number.parseInt(blueHex, 16);
const alpha = 1 - Number.parseInt(alphaHex, 16) / 255;
if (alpha >= 1) {
return {
cssColor: `#${redHex}${greenHex}${blueHex}`.toLowerCase(),
alpha,
};
}
return {
cssColor: `rgba(${red}, ${green}, ${blue}, ${Math.round(alpha * 1000) / 1000})`,
alpha,
};
}

333
apps/web/src/lib/subtitles/build-subtitle-text-element.ts Normal file
View File

@ -0,0 +1,333 @@
import { FONT_SIZE_SCALE_REFERENCE } from "@/constants/text-constants";
import {
getTextVisualRect,
measureTextBlock,
setCanvasLetterSpacing,
} from "@/lib/text/layout";
import { DEFAULTS } from "@/lib/timeline/defaults";
import { TICKS_PER_SECOND } from "@/lib/wasm";
import type { CreateTextElement } from "@/lib/timeline";
import type { SubtitleCue, SubtitleStyleOverrides } from "./types";
const SUBTITLE_MAX_WIDTH_RATIO = 0.8;
const SUBTITLE_BOTTOM_MARGIN_RATIO = 0.05;
const SUBTITLE_FONT_SIZE = 5;
const MEASUREMENT_CANVAS_SIZE = 4096;
function quoteFontFamily({ fontFamily }: { fontFamily: string }): string {
return `"${fontFamily.replace(/"/g, '\\"')}"`;
}
function createMeasurementContext(): CanvasRenderingContext2D | null {
const canvas = document.createElement("canvas");
canvas.width = MEASUREMENT_CANVAS_SIZE;
canvas.height = MEASUREMENT_CANVAS_SIZE;
return canvas.getContext("2d");
}
function measureLineWidth({
ctx,
text,
}: {
ctx: CanvasRenderingContext2D;
text: string;
}): number {
return ctx.measureText(text).width;
}
function wrapSubtitleText({
ctx,
text,
maxWidth,
}: {
ctx: CanvasRenderingContext2D;
text: string;
maxWidth: number;
}): string {
const normalized = text.trim().replace(/\r\n/g, "\n");
const paragraphs = normalized.split("\n");
const wrappedParagraphs: string[] = [];
for (const paragraph of paragraphs) {
const trimmedParagraph = paragraph.trim();
if (!trimmedParagraph) {
wrappedParagraphs.push("");
continue;
}
const words = trimmedParagraph.split(/\s+/);
let currentLine = words[0] ?? "";
const lines: string[] = [];
for (let i = 1; i < words.length; i++) {
const nextLine = `${currentLine} ${words[i]}`;
if (measureLineWidth({ ctx, text: nextLine }) <= maxWidth) {
currentLine = nextLine;
continue;
}
lines.push(currentLine);
currentLine = words[i];
}
lines.push(currentLine);
wrappedParagraphs.push(lines.join("\n"));
}
return wrappedParagraphs.join("\n");
}
function measureWrappedTextBlock({
ctx,
content,
canvasHeight,
textAlign,
background,
fontSize,
lineHeight,
}: {
ctx: CanvasRenderingContext2D;
content: string;
canvasHeight: number;
textAlign: CreateTextElement["textAlign"];
background: CreateTextElement["background"];
fontSize: number;
lineHeight: number;
}) {
const scaledFontSize = fontSize * (canvasHeight / FONT_SIZE_SCALE_REFERENCE);
const lineHeightPx = lineHeight * scaledFontSize;
const lines = content.split("\n");
const lineMetrics = lines.map((line) => ctx.measureText(line));
const block = measureTextBlock({
lineMetrics,
lineHeightPx,
});
const visualRect = getTextVisualRect({
textAlign,
block,
background,
fontSizeRatio: fontSize / DEFAULTS.text.element.fontSize,
});
return {
block,
visualRect,
};
}
function resolveSubtitleStyle({
style,
}: {
style: SubtitleStyleOverrides | undefined;
}): {
fontFamily: string;
fontSize: number;
color: string;
textAlign: CreateTextElement["textAlign"];
fontWeight: CreateTextElement["fontWeight"];
fontStyle: CreateTextElement["fontStyle"];
textDecoration: CreateTextElement["textDecoration"];
letterSpacing: number;
lineHeight: number;
background: CreateTextElement["background"];
placement: NonNullable<SubtitleStyleOverrides["placement"]>;
} {
const fontSize =
style?.fontSizeRatioOfPlayHeight != null
? style.fontSizeRatioOfPlayHeight * FONT_SIZE_SCALE_REFERENCE
: (style?.fontSize ?? SUBTITLE_FONT_SIZE);
return {
fontFamily: style?.fontFamily ?? DEFAULTS.text.element.fontFamily,
fontSize,
color: style?.color ?? DEFAULTS.text.element.color,
textAlign: style?.textAlign ?? "center",
fontWeight: style?.fontWeight ?? "bold",
fontStyle: style?.fontStyle ?? DEFAULTS.text.element.fontStyle,
textDecoration:
style?.textDecoration ?? DEFAULTS.text.element.textDecoration,
letterSpacing: style?.letterSpacing ?? DEFAULTS.text.letterSpacing,
lineHeight: style?.lineHeight ?? DEFAULTS.text.lineHeight,
background: {
...DEFAULTS.text.element.background,
enabled: false,
...(style?.background ?? {}),
},
placement: {
verticalAlign: style?.placement?.verticalAlign ?? "bottom",
marginLeftRatio: style?.placement?.marginLeftRatio,
marginRightRatio: style?.placement?.marginRightRatio,
marginVerticalRatio: style?.placement?.marginVerticalRatio,
},
};
}
function resolveTargetWidth({
canvasWidth,
placement,
}: {
canvasWidth: number;
placement: ReturnType<typeof resolveSubtitleStyle>["placement"];
}): number {
const leftRatio = placement.marginLeftRatio ?? 0;
const rightRatio = placement.marginRightRatio ?? 0;
const hasExplicitMargins = leftRatio > 0 || rightRatio > 0;
if (!hasExplicitMargins) {
return canvasWidth * SUBTITLE_MAX_WIDTH_RATIO;
}
const availableWidth = canvasWidth * (1 - leftRatio - rightRatio);
return Math.max(0, availableWidth);
}
function resolvePositionX({
canvasWidth,
textAlign,
placement,
visualRect,
}: {
canvasWidth: number;
textAlign: CreateTextElement["textAlign"];
placement: ReturnType<typeof resolveSubtitleStyle>["placement"];
visualRect: { left: number; width: number };
}): number {
const leftMargin = canvasWidth * (placement.marginLeftRatio ?? 0);
const rightMargin = canvasWidth * (placement.marginRightRatio ?? 0);
const canvasCenterX = canvasWidth / 2;
if (textAlign === "left") {
return leftMargin - visualRect.left - canvasCenterX;
}
if (textAlign === "right") {
return (
canvasWidth -
rightMargin -
(visualRect.left + visualRect.width) -
canvasCenterX
);
}
const availableWidth = canvasWidth - leftMargin - rightMargin;
const targetCenterX = leftMargin + availableWidth / 2;
return (
targetCenterX - (visualRect.left + visualRect.width / 2) - canvasCenterX
);
}
function resolvePositionY({
canvasHeight,
placement,
visualRect,
}: {
canvasHeight: number;
placement: ReturnType<typeof resolveSubtitleStyle>["placement"];
visualRect: { top: number; height: number };
}): number {
const margin =
canvasHeight *
(placement.marginVerticalRatio ?? SUBTITLE_BOTTOM_MARGIN_RATIO);
const canvasCenterY = canvasHeight / 2;
if (placement.verticalAlign === "top") {
return margin - visualRect.top - canvasCenterY;
}
if (placement.verticalAlign === "middle") {
const targetCenterY = canvasHeight / 2;
return (
targetCenterY - (visualRect.top + visualRect.height / 2) - canvasCenterY
);
}
return (
canvasHeight - margin - (visualRect.top + visualRect.height) - canvasCenterY
);
}
export function buildSubtitleTextElement({
index,
caption,
canvasSize,
}: {
index: number;
caption: SubtitleCue;
canvasSize: { width: number; height: number };
}): CreateTextElement {
const ctx = createMeasurementContext();
const style = resolveSubtitleStyle({
style: caption.style,
});
const fontFamily = quoteFontFamily({
fontFamily: style.fontFamily,
});
const fontWeight = style.fontWeight;
const fontStyle = style.fontStyle === "italic" ? "italic" : "normal";
const scaledFontSize =
style.fontSize * (canvasSize.height / FONT_SIZE_SCALE_REFERENCE);
const fontString = `${fontStyle} ${fontWeight} ${scaledFontSize}px ${fontFamily}, sans-serif`;
const maxWidth = resolveTargetWidth({
canvasWidth: canvasSize.width,
placement: style.placement,
});
let content = caption.text;
let positionX = 0;
let positionY = 0;
if (ctx) {
ctx.font = fontString;
setCanvasLetterSpacing({ ctx, letterSpacingPx: style.letterSpacing });
content = wrapSubtitleText({
ctx,
text: caption.text,
maxWidth,
});
const measurement = measureWrappedTextBlock({
ctx,
content,
canvasHeight: canvasSize.height,
textAlign: style.textAlign,
background: style.background,
fontSize: style.fontSize,
lineHeight: style.lineHeight,
});
positionX = resolvePositionX({
canvasWidth: canvasSize.width,
textAlign: style.textAlign,
placement: style.placement,
visualRect: measurement.visualRect,
});
positionY = resolvePositionY({
canvasHeight: canvasSize.height,
placement: style.placement,
visualRect: measurement.visualRect,
});
}
return {
...DEFAULTS.text.element,
name: `Caption ${index + 1}`,
content,
duration: Math.round(caption.duration * TICKS_PER_SECOND),
startTime: Math.round(caption.startTime * TICKS_PER_SECOND),
fontSize: style.fontSize,
fontFamily: style.fontFamily,
color: style.color,
textAlign: style.textAlign,
fontWeight: style.fontWeight,
fontStyle: style.fontStyle,
textDecoration: style.textDecoration,
letterSpacing: style.letterSpacing,
lineHeight: style.lineHeight,
background: style.background,
transform: {
...DEFAULTS.text.element.transform,
position: {
x: positionX,
y: positionY,
},
},
};
}

40
apps/web/src/lib/subtitles/insert.ts Normal file
View File

@ -0,0 +1,40 @@
import type { EditorCore } from "@/core";
import {
AddTrackCommand,
BatchCommand,
InsertElementCommand,
} from "@/lib/commands";
import { buildSubtitleTextElement } from "./build-subtitle-text-element";
import type { SubtitleCue } from "./types";
export function insertCaptionChunksAsTextTrack({
editor,
captions,
}: {
editor: EditorCore;
captions: SubtitleCue[];
}): string | null {
if (captions.length === 0) {
return null;
}
const addTrackCommand = new AddTrackCommand("text", 0);
const trackId = addTrackCommand.getTrackId();
const canvasSize = editor.project.getActive().settings.canvasSize;
const insertCommands = captions.map(
(caption, index) =>
new InsertElementCommand({
placement: { mode: "explicit", trackId },
element: buildSubtitleTextElement({
index,
caption,
canvasSize,
}),
}),
);
editor.command.execute({
command: new BatchCommand([addTrackCommand, ...insertCommands]),
});
return trackId;
}

28
apps/web/src/lib/subtitles/parse.ts Normal file
View File

@ -0,0 +1,28 @@
import { parseAss } from "./ass";
import { parseSrt } from "./srt";
import type { ParseSubtitleResult } from "./types";
export type { ParseSubtitleResult, SubtitleCue } from "./types";
export function parseSubtitleFile({
fileName,
input,
}: {
fileName: string;
input: string;
}): ParseSubtitleResult {
const extension = getFileExtension({ fileName });
switch (extension) {
case "srt":
return parseSrt({ input });
case "ass":
return parseAss({ input });
default:
throw new Error("Unsupported subtitle format");
}
}
function getFileExtension({ fileName }: { fileName: string }): string {
const extension = fileName.split(".").pop();
return extension?.toLowerCase() ?? "";
}

98
apps/web/src/lib/subtitles/srt.ts Normal file
View File

@ -0,0 +1,98 @@
import type { ParseSubtitleResult, SubtitleCue } from "./types";
const TIMESTAMP_SEPARATOR = /\s*-->\s*/;
const TIMESTAMP_PATTERN =
/^(\d{2}:\d{2}:\d{2}[,.]\d{1,3})\s*-->\s*(\d{2}:\d{2}:\d{2}[,.]\d{1,3})/;
export function parseSrt({ input }: { input: string }): ParseSubtitleResult {
const normalized = input.replace(/\r\n?/g, "\n").trim();
if (!normalized) {
return {
captions: [],
skippedCueCount: 0,
warnings: [],
};
}
const blocks = normalized.split(/\n{2,}/);
const cues: SubtitleCue[] = [];
let skippedCueCount = 0;
for (const block of blocks) {
const lines = block
.split("\n")
.map((line) => line.trim())
.filter((line) => line.length > 0);
if (lines.length < 2) {
skippedCueCount += 1;
continue;
}
const timestampIndex = TIMESTAMP_SEPARATOR.test(lines[0]) ? 0 : 1;
const timestampLine = lines[timestampIndex];
if (!timestampLine || !TIMESTAMP_PATTERN.test(timestampLine)) {
skippedCueCount += 1;
continue;
}
const textLines = lines.slice(timestampIndex + 1);
const text = textLines.join("\n").trim();
if (!text) {
skippedCueCount += 1;
continue;
}
const [rawStart, rawEnd] = timestampLine.split(TIMESTAMP_SEPARATOR);
if (!rawStart || !rawEnd) {
skippedCueCount += 1;
continue;
}
const startTime = parseSrtTimestamp({ input: rawStart });
const endTime = parseSrtTimestamp({ input: rawEnd });
const duration = endTime - startTime;
if (
!Number.isFinite(startTime) ||
!Number.isFinite(endTime) ||
duration <= 0
) {
skippedCueCount += 1;
continue;
}
cues.push({
text,
startTime,
duration,
});
}
return {
captions: cues,
skippedCueCount,
warnings: [],
};
}
function parseSrtTimestamp({ input }: { input: string }): number {
const normalized = input.trim().replace(",", ".");
const match = normalized.match(/^(\d{2}):(\d{2}):(\d{2})\.(\d{1,3})$/);
if (!match) {
return Number.NaN;
}
const [, hours, minutes, seconds, milliseconds] = match;
const parsedHours = Number.parseInt(hours, 10);
const parsedMinutes = Number.parseInt(minutes, 10);
const parsedSeconds = Number.parseInt(seconds, 10);
const parsedMilliseconds = Number.parseInt(milliseconds.padEnd(3, "0"), 10);
return (
parsedHours * 3600 +
parsedMinutes * 60 +
parsedSeconds +
parsedMilliseconds / 1000
);
}

46
apps/web/src/lib/subtitles/types.ts Normal file
View File

@ -0,0 +1,46 @@
import type { TextBackground, TextElement } from "@/lib/timeline";
import type { CaptionChunk } from "@/lib/transcription/types";
export interface SubtitlePlacementStyle {
verticalAlign?: "top" | "middle" | "bottom";
marginLeftRatio?: number;
marginRightRatio?: number;
marginVerticalRatio?: number;
}
export interface SubtitleStyleOverrides {
/**
* Font size in app units (same coordinate space as TextElement.fontSize).
* Use fontSizeRatioOfPlayHeight when the source coordinate space is unknown
* (e.g. ASS files, where font size is relative to the script's play resolution).
*/
fontSize?: number;
/**
* Font size expressed as a fraction of the reference canvas height.
* Set by the ASS parser so the builder can convert to app units without
* the parser needing to know about the app's coordinate system.
* Takes precedence over fontSize when both are present.
*/
fontSizeRatioOfPlayHeight?: number;
fontFamily?: string;
color?: string;
background?: Pick<TextBackground, "enabled" | "color"> &
Partial<Omit<TextBackground, "enabled" | "color">>;
textAlign?: TextElement["textAlign"];
fontWeight?: TextElement["fontWeight"];
fontStyle?: TextElement["fontStyle"];
textDecoration?: TextElement["textDecoration"];
letterSpacing?: number;
lineHeight?: number;
placement?: SubtitlePlacementStyle;
}
export interface SubtitleCue extends CaptionChunk {
style?: SubtitleStyleOverrides;
}
export interface ParseSubtitleResult {
captions: SubtitleCue[];
skippedCueCount: number;
warnings: string[];
}