diff --git a/apps/web/src/components/editor/panels/assets/views/captions.tsx b/apps/web/src/components/editor/panels/assets/views/captions.tsx index c89e587b..488875a8 100644 --- a/apps/web/src/components/editor/panels/assets/views/captions.tsx +++ b/apps/web/src/components/editor/panels/assets/views/captions.tsx @@ -1,179 +1,282 @@ -import { Button } from "@/components/ui/button"; -import { PanelView } from "@/components/editor/panels/assets/views/base-panel"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@/components/ui/select"; -import { useState, useRef } from "react"; -import { extractTimelineAudio } from "@/lib/media/mediabunny"; -import { useEditor } from "@/hooks/use-editor"; -import { - DEFAULT_TRANSCRIPTION_SAMPLE_RATE, - TRANSCRIPTION_LANGUAGES, -} from "@/constants/transcription-constants"; -import type { - TranscriptionLanguage, - TranscriptionProgress, -} from "@/lib/transcription/types"; -import { transcriptionService } from "@/services/transcription/service"; -import { decodeAudioToFloat32 } from "@/lib/media/audio"; -import { buildCaptionChunks } from "@/lib/transcription/caption"; -import { Spinner } from "@/components/ui/spinner"; -import { - Section, - SectionContent, - SectionField, - SectionFields, -} from "@/components/section"; -import { DEFAULTS } from "@/lib/timeline/defaults"; -import { - AddTrackCommand, - BatchCommand, - InsertElementCommand, -} from "@/lib/commands"; - -export function Captions() { - const [selectedLanguage, setSelectedLanguage] = - useState("auto"); - const [isProcessing, setIsProcessing] = useState(false); - const [processingStep, setProcessingStep] = useState(""); - const [error, setError] = useState(null); - const containerRef = useRef(null); - const editor = useEditor(); - - const handleProgress = (progress: TranscriptionProgress) => { - if (progress.status === "loading-model") { - setProcessingStep(`Loading model ${Math.round(progress.progress)}%`); - } else if (progress.status === "transcribing") { - setProcessingStep("Transcribing..."); - } - }; - - const handleGenerateTranscript = async () => { - try { - setIsProcessing(true); - setError(null); - setProcessingStep("Extracting audio..."); - - const audioBlob = await extractTimelineAudio({ - tracks: editor.scenes.getActiveScene().tracks, - mediaAssets: editor.media.getAssets(), - totalDuration: editor.timeline.getTotalDuration(), - }); - - setProcessingStep("Preparing audio..."); - const { samples } = await decodeAudioToFloat32({ - audioBlob, - sampleRate: DEFAULT_TRANSCRIPTION_SAMPLE_RATE, - }); - - const result = await transcriptionService.transcribe({ - audioData: samples, - language: selectedLanguage === "auto" ? undefined : selectedLanguage, - onProgress: handleProgress, - }); - - setProcessingStep("Generating captions..."); - const captionChunks = buildCaptionChunks({ segments: result.segments }); - - const addTrackCommand = new AddTrackCommand("text", 0); - const insertCommands = captionChunks.map( - (caption, i) => - new InsertElementCommand({ - placement: { - mode: "explicit", - trackId: addTrackCommand.getTrackId(), - }, - element: { - ...DEFAULTS.text.element, - name: `Caption ${i + 1}`, - content: caption.text, - duration: caption.duration, - startTime: caption.startTime, - fontSize: 65, - fontWeight: "bold", - }, - }), - ); - - editor.command.execute({ - command: new BatchCommand([addTrackCommand, ...insertCommands]), - }); - } catch (error) { - console.error("Transcription failed:", error); - setError( - error instanceof Error ? error.message : "An unexpected error occurred", - ); - } finally { - setIsProcessing(false); - setProcessingStep(""); - } - }; - - const handleLanguageChange = ({ value }: { value: string }) => { - if (value === "auto") { - setSelectedLanguage("auto"); - return; - } - - const matchedLanguage = TRANSCRIPTION_LANGUAGES.find( - (language) => language.code === value, - ); - if (!matchedLanguage) return; - setSelectedLanguage(matchedLanguage.code); - }; - - return ( - -
- - - - - - - - {error && ( -
-

{error}

-
- )} - -
-
- - - -
- - ); -} +import { Button } from "@/components/ui/button"; +import { PanelView } from "@/components/editor/panels/assets/views/base-panel"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { useReducer, useRef, useState } from "react"; +import { extractTimelineAudio } from "@/lib/media/mediabunny"; +import { useEditor } from "@/hooks/use-editor"; +import { + DEFAULT_TRANSCRIPTION_SAMPLE_RATE, + TRANSCRIPTION_LANGUAGES, +} from "@/constants/transcription-constants"; +import type { + CaptionChunk, + TranscriptionLanguage, + TranscriptionProgress, +} from "@/lib/transcription/types"; +import { transcriptionService } from "@/services/transcription/service"; +import { decodeAudioToFloat32 } from "@/lib/media/audio"; +import { buildCaptionChunks } from "@/lib/transcription/caption"; +import { insertCaptionChunksAsTextTrack } from "@/lib/subtitles/insert"; +import { parseSubtitleFile } from "@/lib/subtitles/parse"; +import { Spinner } from "@/components/ui/spinner"; +import { + Section, + SectionContent, + SectionField, + SectionFields, +} from "@/components/section"; +import { CloudUploadIcon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; + +type ProcessingState = + | { status: "idle"; error: string | null; warnings: string[] } + | { status: "processing"; step: string }; + +type ProcessingAction = + | { type: "start"; step: string } + | { type: "update_step"; step: string } + | { type: "succeed"; warnings: string[] } + | { type: "fail"; error: string }; + +const IDLE_STATE: ProcessingState = { status: "idle", error: null, warnings: [] }; + +function processingReducer( + state: ProcessingState, + action: ProcessingAction, +): ProcessingState { + switch (action.type) { + case "start": + return { status: "processing", step: action.step }; + case "update_step": + if (state.status !== "processing") return state; + return { status: "processing", step: action.step }; + case "succeed": + return { status: "idle", error: null, warnings: action.warnings }; + case "fail": + return { status: "idle", error: action.error, warnings: [] }; + } +} + +export function Captions() { + const [selectedLanguage, setSelectedLanguage] = + useState("auto"); + const [processing, dispatch] = useReducer(processingReducer, IDLE_STATE); + const containerRef = useRef(null); + const fileInputRef = useRef(null); + const editor = useEditor(); + + const isProcessing = processing.status === "processing"; + + const handleProgress = (progress: TranscriptionProgress) => { + if (progress.status === "loading-model") { + dispatch({ + type: "update_step", + step: `Loading model ${Math.round(progress.progress)}%`, + }); + } else if (progress.status === "transcribing") { + dispatch({ type: "update_step", step: "Transcribing..." }); + } + }; + + const insertCaptions = ({ captions }: { captions: CaptionChunk[] }): boolean => { + const trackId = insertCaptionChunksAsTextTrack({ editor, captions }); + return trackId !== null; + }; + + const handleGenerateTranscript = async () => { + dispatch({ type: "start", step: "Extracting audio..." }); + try { + const audioBlob = await extractTimelineAudio({ + tracks: editor.scenes.getActiveScene().tracks, + mediaAssets: editor.media.getAssets(), + totalDuration: editor.timeline.getTotalDuration(), + }); + + dispatch({ type: "update_step", step: "Preparing audio..." }); + const { samples } = await decodeAudioToFloat32({ + audioBlob, + sampleRate: DEFAULT_TRANSCRIPTION_SAMPLE_RATE, + }); + + const result = await transcriptionService.transcribe({ + audioData: samples, + language: selectedLanguage === "auto" ? undefined : selectedLanguage, + onProgress: handleProgress, + }); + + dispatch({ type: "update_step", step: "Generating captions..." }); + const captionChunks = buildCaptionChunks({ segments: result.segments }); + + if (!insertCaptions({ captions: captionChunks })) { + dispatch({ type: "fail", error: "No captions were generated" }); + return; + } + + dispatch({ type: "succeed", warnings: [] }); + } catch (error) { + console.error("Transcription failed:", error); + dispatch({ + type: "fail", + error: error instanceof Error ? error.message : "An unexpected error occurred", + }); + } + }; + + const handleImportClick = () => { + fileInputRef.current?.click(); + }; + + const handleImportFile = async ({ file }: { file: File }) => { + dispatch({ type: "start", step: "Reading subtitle file..." }); + try { + const input = await file.text(); + const result = parseSubtitleFile({ + fileName: file.name, + input, + }); + + if (result.captions.length === 0) { + dispatch({ + type: "fail", + error: "No valid subtitle cues were found in the subtitle file", + }); + return; + } + + dispatch({ type: "update_step", step: "Importing subtitles..." }); + + if (!insertCaptions({ captions: result.captions })) { + dispatch({ type: "fail", error: "No captions were generated" }); + return; + } + + const nextWarnings = [...result.warnings]; + if (result.skippedCueCount > 0) { + nextWarnings.unshift( + `Imported ${result.captions.length} subtitle cue(s) and skipped ${result.skippedCueCount} malformed cue(s).`, + ); + } + + dispatch({ type: "succeed", warnings: nextWarnings }); + } catch (error) { + console.error("Subtitle import failed:", error); + dispatch({ + type: "fail", + error: error instanceof Error ? error.message : "An unexpected error occurred", + }); + } + }; + + const handleFileChange = async ({ + event, + }: { + event: React.ChangeEvent; + }) => { + const file = event.target.files?.[0]; + if (event.target) { + event.target.value = ""; + } + if (!file) return; + + await handleImportFile({ file }); + }; + + const handleLanguageChange = ({ value }: { value: string }) => { + if (value === "auto") { + setSelectedLanguage("auto"); + return; + } + + const matchedLanguage = TRANSCRIPTION_LANGUAGES.find( + (language) => language.code === value, + ); + if (!matchedLanguage) return; + setSelectedLanguage(matchedLanguage.code); + }; + + const error = processing.status === "idle" ? processing.error : null; + const warnings = processing.status === "idle" ? processing.warnings : []; + + return ( + + + Import + + } + ref={containerRef} + > + void handleFileChange({ event })} + /> +
+ + + + + + + + + {error && ( +
+

{error}

+
+ )} + {warnings.length > 0 && ( +
+
    + {warnings.map((warning) => ( +
  • {warning}
    • + ))} +
+
+ )} + +
+ + ); +} diff --git a/apps/web/src/lib/subtitles/ass.ts b/apps/web/src/lib/subtitles/ass.ts new file mode 100644 index 00000000..030c5b36 --- /dev/null +++ b/apps/web/src/lib/subtitles/ass.ts @@ -0,0 +1,514 @@ +import type { + ParseSubtitleResult, + SubtitleCue, + SubtitleStyleOverrides, +} from "./types"; + +const ASS_DEFAULT_PLAY_RES_X = 384; +const ASS_DEFAULT_PLAY_RES_Y = 288; +const STYLE_SECTION_NAMES = new Set(["v4 styles", "v4+ styles"]); +const ALIGNMENT_MAP: Record< + number, + { + textAlign: NonNullable; + verticalAlign: NonNullable< + NonNullable["verticalAlign"] + >; + } +> = { + 1: { textAlign: "left", verticalAlign: "bottom" }, + 2: { textAlign: "center", verticalAlign: "bottom" }, + 3: { textAlign: "right", verticalAlign: "bottom" }, + 4: { textAlign: "left", verticalAlign: "middle" }, + 5: { textAlign: "center", verticalAlign: "middle" }, + 6: { textAlign: "right", verticalAlign: "middle" }, + 7: { textAlign: "left", verticalAlign: "top" }, + 8: { textAlign: "center", verticalAlign: "top" }, + 9: { textAlign: "right", verticalAlign: "top" }, +}; + +interface AssScriptInfo { + playResX: number; + playResY: number; +} + +interface AssStyleRecord { + name: string; + fontname?: string; + fontsize?: string; + primarycolour?: string; + secondarycolour?: string; + outlinecolour?: string; + backcolour?: string; + bold?: string; + italic?: string; + underline?: string; + strikeout?: string; + scalex?: string; + scaley?: string; + spacing?: string; + angle?: string; + borderstyle?: string; + outline?: string; + shadow?: string; + alignment?: string; + marginl?: string; + marginr?: string; + marginv?: string; +} + +export function parseAss({ input }: { input: string }): ParseSubtitleResult { + const normalized = input.replace(/\r\n?/g, "\n").trim(); + if (!normalized) { + return { + captions: [], + skippedCueCount: 0, + warnings: [], + }; + } + + const scriptInfo: AssScriptInfo = { + playResX: ASS_DEFAULT_PLAY_RES_X, + playResY: ASS_DEFAULT_PLAY_RES_Y, + }; + const warnings = new Set(); + const styles = new Map(); + const captions: SubtitleCue[] = []; + + let currentSection = ""; + let styleFormat: string[] | null = null; + let eventFormat: string[] | null = null; + let skippedCueCount = 0; + let strippedInlineTagCueCount = 0; + let skippedNonDialogueEventCount = 0; + let ignoredEffectCount = 0; + let missingStyleCueCount = 0; + let usesHeavilyUnsupportedStyles = false; + + for (const rawLine of normalized.split("\n")) { + const line = rawLine.trim(); + if (!line || line.startsWith(";")) { + continue; + } + + const sectionMatch = line.match(/^\[(.+)\]$/); + if (sectionMatch) { + currentSection = sectionMatch[1].trim().toLowerCase(); + continue; + } + + if (currentSection === "script info") { + const [rawKey, ...rest] = line.split(":"); + if (!rawKey || rest.length === 0) continue; + + const key = rawKey.trim().toLowerCase(); + const value = rest.join(":").trim(); + const parsedValue = parseFloat(value); + + if (!Number.isFinite(parsedValue) || parsedValue <= 0) { + continue; + } + + if (key === "playresx") { + scriptInfo.playResX = parsedValue; + } + if (key === "playresy") { + scriptInfo.playResY = parsedValue; + } + continue; + } + + if (STYLE_SECTION_NAMES.has(currentSection)) { + if (line.startsWith("Format:")) { + styleFormat = parseAssFormat({ line }); + continue; + } + + if (line.startsWith("Style:") && styleFormat) { + const values = splitAssFields({ + value: line.slice("Style:".length).trim(), + expectedFieldCount: styleFormat.length, + }); + if (values.length !== styleFormat.length) { + continue; + } + + const record = mapFieldsToRecord({ + fields: styleFormat, + values, + }); + if (!record.name) { + continue; + } + + const mappedStyle = mapAssStyleToSubtitleStyle({ + style: record, + scriptInfo, + }); + usesHeavilyUnsupportedStyles ||= mappedStyle.hasUnsupportedFeatures; + styles.set(record.name.toLowerCase(), mappedStyle.style); + } + continue; + } + + if (currentSection !== "events") { + continue; + } + + if (line.startsWith("Format:")) { + eventFormat = parseAssFormat({ line }); + continue; + } + + if (!eventFormat) { + continue; + } + + if (line.startsWith("Dialogue:")) { + const values = splitAssFields({ + value: line.slice("Dialogue:".length).trim(), + expectedFieldCount: eventFormat.length, + }); + if (values.length !== eventFormat.length) { + skippedCueCount += 1; + continue; + } + + const record = mapFieldsToRecord>({ + fields: eventFormat, + values, + }); + const startTime = parseAssTimestamp({ input: record.start }); + const endTime = parseAssTimestamp({ input: record.end }); + const duration = endTime - startTime; + if ( + !Number.isFinite(startTime) || + !Number.isFinite(endTime) || + duration <= 0 + ) { + skippedCueCount += 1; + continue; + } + + const strippedText = stripAssText({ + input: record.text ?? "", + }); + strippedInlineTagCueCount += strippedText.hadInlineTags ? 1 : 0; + if (!strippedText.text) { + skippedCueCount += 1; + continue; + } + + if (record.effect?.trim()) { + ignoredEffectCount += 1; + } + + const referencedStyleName = record.style?.trim().toLowerCase(); + const referencedStyle = referencedStyleName + ? styles.get(referencedStyleName) + : undefined; + if (referencedStyleName && !referencedStyle) { + missingStyleCueCount += 1; + } + const style = referencedStyle ?? styles.get("default"); + + captions.push({ + text: strippedText.text, + startTime, + duration, + style, + }); + continue; + } + + if (line.includes(":")) { + skippedNonDialogueEventCount += 1; + } + } + + if (strippedInlineTagCueCount > 0) { + warnings.add( + `Stripped unsupported ASS inline override tags from ${strippedInlineTagCueCount} subtitle cue(s).`, + ); + } + + if (ignoredEffectCount > 0) { + warnings.add( + `Ignored ASS event effects in ${ignoredEffectCount} subtitle cue(s).`, + ); + } + + if (missingStyleCueCount > 0) { + warnings.add( + `Fell back to default subtitle styling for ${missingStyleCueCount} cue(s) that referenced missing ASS styles.`, + ); + } + + if (skippedNonDialogueEventCount > 0) { + warnings.add( + `Ignored ${skippedNonDialogueEventCount} non-dialogue ASS event(s).`, + ); + } + + if (usesHeavilyUnsupportedStyles) { + warnings.add( + "Ignored unsupported ASS style features such as outline, shadow, rotation, or scaling.", + ); + } + + return { + captions, + skippedCueCount, + warnings: [...warnings], + }; +} + +function parseAssFormat({ line }: { line: string }): string[] { + return line + .slice(line.indexOf(":") + 1) + .split(",") + .map((field) => field.trim().toLowerCase()); +} + +function splitAssFields({ + value, + expectedFieldCount, +}: { + value: string; + expectedFieldCount: number; +}): string[] { + if (expectedFieldCount <= 1) { + return [value]; + } + + const result: string[] = []; + let current = ""; + + for (const character of value) { + if (character === "," && result.length < expectedFieldCount - 1) { + result.push(current.trim()); + current = ""; + continue; + } + + current += character; + } + + result.push(current.trim()); + return result; +} + +function mapFieldsToRecord({ + fields, + values, +}: { + fields: string[]; + values: string[]; +}): T { + const record = {} as Record; + + for (let index = 0; index < fields.length; index++) { + record[fields[index]] = values[index]; + } + + return record as T; +} + +function parseAssTimestamp({ input }: { input: string | undefined }): number { + if (!input) { + return Number.NaN; + } + + const match = input.trim().match(/^(\d+):(\d{2}):(\d{2})[.](\d{1,2}|\d{3})$/); + if (!match) { + return Number.NaN; + } + + const [, hours, minutes, seconds, fraction] = match; + const parsedHours = Number.parseInt(hours, 10); + const parsedMinutes = Number.parseInt(minutes, 10); + const parsedSeconds = Number.parseInt(seconds, 10); + const parsedMilliseconds = Number.parseInt(fraction.padEnd(3, "0"), 10); + + return ( + parsedHours * 3600 + + parsedMinutes * 60 + + parsedSeconds + + parsedMilliseconds / 1000 + ); +} + +function stripAssText({ input }: { input: string }): { + text: string; + hadInlineTags: boolean; +} { + const hadInlineTags = /\{[^}]*\}/.test(input); + const text = input + .replace(/\{[^}]*\}/g, "") + .replace(/\\N/gi, "\n") + .replace(/\\h/g, " ") + .replace(/\\n/g, "\n") + .trim(); + + return { + text, + hadInlineTags, + }; +} + +function mapAssStyleToSubtitleStyle({ + style, + scriptInfo, +}: { + style: AssStyleRecord; + scriptInfo: AssScriptInfo; +}): { + style: SubtitleStyleOverrides; + hasUnsupportedFeatures: boolean; +} { + const fontSize = parseFloat(style.fontsize ?? ""); + const primaryColor = parseAssColor({ input: style.primarycolour }); + const backColor = parseAssColor({ input: style.backcolour }); + const bold = parseAssBoolean({ input: style.bold }); + const italic = parseAssBoolean({ input: style.italic }); + const underline = parseAssBoolean({ input: style.underline }); + const strikeOut = parseAssBoolean({ input: style.strikeout }); + const borderStyle = parseFloat(style.borderstyle ?? ""); + const spacing = parseFloat(style.spacing ?? ""); + const alignment = parseFloat(style.alignment ?? ""); + const marginLeft = parseFloat(style.marginl ?? ""); + const marginRight = parseFloat(style.marginr ?? ""); + const marginVertical = parseFloat(style.marginv ?? ""); + // Store as a ratio of playResY so the builder can convert to app units + // without the parser needing to know the app's coordinate system. + const fontSizeRatioOfPlayHeight = Number.isFinite(fontSize) + ? Math.round((fontSize / scriptInfo.playResY) * 1000) / 1000 + : null; + + const mappedAlignment = + ALIGNMENT_MAP[Math.round(alignment)] ?? ALIGNMENT_MAP[2]; + const placement = + Number.isFinite(marginLeft) || + Number.isFinite(marginRight) || + Number.isFinite(marginVertical) || + mappedAlignment.verticalAlign !== "bottom" + ? { + verticalAlign: mappedAlignment.verticalAlign, + marginLeftRatio: Number.isFinite(marginLeft) + ? marginLeft / scriptInfo.playResX + : undefined, + marginRightRatio: Number.isFinite(marginRight) + ? marginRight / scriptInfo.playResX + : undefined, + marginVerticalRatio: Number.isFinite(marginVertical) + ? marginVertical / scriptInfo.playResY + : undefined, + } + : undefined; + + const styleOverrides: SubtitleStyleOverrides = { + ...(style.fontname ? { fontFamily: style.fontname.trim() } : {}), + ...(fontSizeRatioOfPlayHeight !== null && fontSizeRatioOfPlayHeight > 0 + ? { fontSizeRatioOfPlayHeight } + : {}), + ...(primaryColor?.cssColor ? { color: primaryColor.cssColor } : {}), + ...(bold !== null ? { fontWeight: bold ? "bold" : "normal" } : {}), + ...(italic !== null ? { fontStyle: italic ? "italic" : "normal" } : {}), + ...(underline || strikeOut + ? { + textDecoration: underline + ? "underline" + : strikeOut + ? "line-through" + : "none", + } + : {}), + ...(Number.isFinite(spacing) ? { letterSpacing: spacing } : {}), + textAlign: mappedAlignment.textAlign, + ...(placement ? { placement } : {}), + ...(backColor?.cssColor && Math.round(borderStyle) === 3 + ? { + background: { + enabled: backColor.alpha > 0, + color: backColor.alpha > 0 ? backColor.cssColor : "transparent", + }, + } + : {}), + }; + + const hasUnsupportedFeatures = + Math.round(borderStyle) !== 1 && Math.round(borderStyle) !== 3 + ? true + : (parseFloat(style.outline ?? "") || 0) > 0 || + (parseFloat(style.shadow ?? "") || 0) > 0 || + (parseFloat(style.angle ?? "") || 0) !== 0 || + (Number.isFinite(parseFloat(style.scalex ?? "")) && + parseFloat(style.scalex ?? "") !== 100) || + (Number.isFinite(parseFloat(style.scaley ?? "")) && + parseFloat(style.scaley ?? "") !== 100) || + Boolean(underline && strikeOut); + + return { + style: styleOverrides, + hasUnsupportedFeatures, + }; +} + +function parseAssBoolean({ + input, +}: { + input: string | undefined; +}): boolean | null { + if (input === undefined) { + return null; + } + + const trimmed = input.trim(); + if (!trimmed) { + return null; + } + + const parsed = Number.parseInt(trimmed, 10); + if (!Number.isFinite(parsed)) { + return null; + } + + return parsed !== 0; +} + +function parseAssColor({ + input, +}: { + input: string | undefined; +}): { cssColor: string; alpha: number } | null { + if (!input) { + return null; + } + + const normalized = input.trim().replace(/^&?H/i, "").padStart(8, "0"); + if (!/^[0-9a-fA-F]{8}$/.test(normalized)) { + return null; + } + + const alphaHex = normalized.slice(0, 2); + const blueHex = normalized.slice(2, 4); + const greenHex = normalized.slice(4, 6); + const redHex = normalized.slice(6, 8); + + const red = Number.parseInt(redHex, 16); + const green = Number.parseInt(greenHex, 16); + const blue = Number.parseInt(blueHex, 16); + const alpha = 1 - Number.parseInt(alphaHex, 16) / 255; + + if (alpha >= 1) { + return { + cssColor: `#${redHex}${greenHex}${blueHex}`.toLowerCase(), + alpha, + }; + } + + return { + cssColor: `rgba(${red}, ${green}, ${blue}, ${Math.round(alpha * 1000) / 1000})`, + alpha, + }; +} diff --git a/apps/web/src/lib/subtitles/build-subtitle-text-element.ts b/apps/web/src/lib/subtitles/build-subtitle-text-element.ts new file mode 100644 index 00000000..8be93e9d --- /dev/null +++ b/apps/web/src/lib/subtitles/build-subtitle-text-element.ts @@ -0,0 +1,333 @@ +import { FONT_SIZE_SCALE_REFERENCE } from "@/constants/text-constants"; +import { + getTextVisualRect, + measureTextBlock, + setCanvasLetterSpacing, +} from "@/lib/text/layout"; +import { DEFAULTS } from "@/lib/timeline/defaults"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; +import type { CreateTextElement } from "@/lib/timeline"; +import type { SubtitleCue, SubtitleStyleOverrides } from "./types"; + +const SUBTITLE_MAX_WIDTH_RATIO = 0.8; +const SUBTITLE_BOTTOM_MARGIN_RATIO = 0.05; +const SUBTITLE_FONT_SIZE = 5; +const MEASUREMENT_CANVAS_SIZE = 4096; + +function quoteFontFamily({ fontFamily }: { fontFamily: string }): string { + return `"${fontFamily.replace(/"/g, '\\"')}"`; +} + +function createMeasurementContext(): CanvasRenderingContext2D | null { + const canvas = document.createElement("canvas"); + canvas.width = MEASUREMENT_CANVAS_SIZE; + canvas.height = MEASUREMENT_CANVAS_SIZE; + return canvas.getContext("2d"); +} + +function measureLineWidth({ + ctx, + text, +}: { + ctx: CanvasRenderingContext2D; + text: string; +}): number { + return ctx.measureText(text).width; +} + +function wrapSubtitleText({ + ctx, + text, + maxWidth, +}: { + ctx: CanvasRenderingContext2D; + text: string; + maxWidth: number; +}): string { + const normalized = text.trim().replace(/\r\n/g, "\n"); + const paragraphs = normalized.split("\n"); + const wrappedParagraphs: string[] = []; + + for (const paragraph of paragraphs) { + const trimmedParagraph = paragraph.trim(); + if (!trimmedParagraph) { + wrappedParagraphs.push(""); + continue; + } + + const words = trimmedParagraph.split(/\s+/); + let currentLine = words[0] ?? ""; + const lines: string[] = []; + + for (let i = 1; i < words.length; i++) { + const nextLine = `${currentLine} ${words[i]}`; + if (measureLineWidth({ ctx, text: nextLine }) <= maxWidth) { + currentLine = nextLine; + continue; + } + + lines.push(currentLine); + currentLine = words[i]; + } + + lines.push(currentLine); + wrappedParagraphs.push(lines.join("\n")); + } + + return wrappedParagraphs.join("\n"); +} + +function measureWrappedTextBlock({ + ctx, + content, + canvasHeight, + textAlign, + background, + fontSize, + lineHeight, +}: { + ctx: CanvasRenderingContext2D; + content: string; + canvasHeight: number; + textAlign: CreateTextElement["textAlign"]; + background: CreateTextElement["background"]; + fontSize: number; + lineHeight: number; +}) { + const scaledFontSize = fontSize * (canvasHeight / FONT_SIZE_SCALE_REFERENCE); + const lineHeightPx = lineHeight * scaledFontSize; + const lines = content.split("\n"); + const lineMetrics = lines.map((line) => ctx.measureText(line)); + + const block = measureTextBlock({ + lineMetrics, + lineHeightPx, + }); + const visualRect = getTextVisualRect({ + textAlign, + block, + background, + fontSizeRatio: fontSize / DEFAULTS.text.element.fontSize, + }); + + return { + block, + visualRect, + }; +} + +function resolveSubtitleStyle({ + style, +}: { + style: SubtitleStyleOverrides | undefined; +}): { + fontFamily: string; + fontSize: number; + color: string; + textAlign: CreateTextElement["textAlign"]; + fontWeight: CreateTextElement["fontWeight"]; + fontStyle: CreateTextElement["fontStyle"]; + textDecoration: CreateTextElement["textDecoration"]; + letterSpacing: number; + lineHeight: number; + background: CreateTextElement["background"]; + placement: NonNullable; +} { + const fontSize = + style?.fontSizeRatioOfPlayHeight != null + ? style.fontSizeRatioOfPlayHeight * FONT_SIZE_SCALE_REFERENCE + : (style?.fontSize ?? SUBTITLE_FONT_SIZE); + + return { + fontFamily: style?.fontFamily ?? DEFAULTS.text.element.fontFamily, + fontSize, + color: style?.color ?? DEFAULTS.text.element.color, + textAlign: style?.textAlign ?? "center", + fontWeight: style?.fontWeight ?? "bold", + fontStyle: style?.fontStyle ?? DEFAULTS.text.element.fontStyle, + textDecoration: + style?.textDecoration ?? DEFAULTS.text.element.textDecoration, + letterSpacing: style?.letterSpacing ?? DEFAULTS.text.letterSpacing, + lineHeight: style?.lineHeight ?? DEFAULTS.text.lineHeight, + background: { + ...DEFAULTS.text.element.background, + enabled: false, + ...(style?.background ?? {}), + }, + placement: { + verticalAlign: style?.placement?.verticalAlign ?? "bottom", + marginLeftRatio: style?.placement?.marginLeftRatio, + marginRightRatio: style?.placement?.marginRightRatio, + marginVerticalRatio: style?.placement?.marginVerticalRatio, + }, + }; +} + +function resolveTargetWidth({ + canvasWidth, + placement, +}: { + canvasWidth: number; + placement: ReturnType["placement"]; +}): number { + const leftRatio = placement.marginLeftRatio ?? 0; + const rightRatio = placement.marginRightRatio ?? 0; + const hasExplicitMargins = leftRatio > 0 || rightRatio > 0; + if (!hasExplicitMargins) { + return canvasWidth * SUBTITLE_MAX_WIDTH_RATIO; + } + + const availableWidth = canvasWidth * (1 - leftRatio - rightRatio); + return Math.max(0, availableWidth); +} + +function resolvePositionX({ + canvasWidth, + textAlign, + placement, + visualRect, +}: { + canvasWidth: number; + textAlign: CreateTextElement["textAlign"]; + placement: ReturnType["placement"]; + visualRect: { left: number; width: number }; +}): number { + const leftMargin = canvasWidth * (placement.marginLeftRatio ?? 0); + const rightMargin = canvasWidth * (placement.marginRightRatio ?? 0); + const canvasCenterX = canvasWidth / 2; + + if (textAlign === "left") { + return leftMargin - visualRect.left - canvasCenterX; + } + + if (textAlign === "right") { + return ( + canvasWidth - + rightMargin - + (visualRect.left + visualRect.width) - + canvasCenterX + ); + } + + const availableWidth = canvasWidth - leftMargin - rightMargin; + const targetCenterX = leftMargin + availableWidth / 2; + return ( + targetCenterX - (visualRect.left + visualRect.width / 2) - canvasCenterX + ); +} + +function resolvePositionY({ + canvasHeight, + placement, + visualRect, +}: { + canvasHeight: number; + placement: ReturnType["placement"]; + visualRect: { top: number; height: number }; +}): number { + const margin = + canvasHeight * + (placement.marginVerticalRatio ?? SUBTITLE_BOTTOM_MARGIN_RATIO); + const canvasCenterY = canvasHeight / 2; + + if (placement.verticalAlign === "top") { + return margin - visualRect.top - canvasCenterY; + } + + if (placement.verticalAlign === "middle") { + const targetCenterY = canvasHeight / 2; + return ( + targetCenterY - (visualRect.top + visualRect.height / 2) - canvasCenterY + ); + } + + return ( + canvasHeight - margin - (visualRect.top + visualRect.height) - canvasCenterY + ); +} + +export function buildSubtitleTextElement({ + index, + caption, + canvasSize, +}: { + index: number; + caption: SubtitleCue; + canvasSize: { width: number; height: number }; +}): CreateTextElement { + const ctx = createMeasurementContext(); + const style = resolveSubtitleStyle({ + style: caption.style, + }); + const fontFamily = quoteFontFamily({ + fontFamily: style.fontFamily, + }); + const fontWeight = style.fontWeight; + const fontStyle = style.fontStyle === "italic" ? "italic" : "normal"; + const scaledFontSize = + style.fontSize * (canvasSize.height / FONT_SIZE_SCALE_REFERENCE); + const fontString = `${fontStyle} ${fontWeight} ${scaledFontSize}px ${fontFamily}, sans-serif`; + const maxWidth = resolveTargetWidth({ + canvasWidth: canvasSize.width, + placement: style.placement, + }); + + let content = caption.text; + let positionX = 0; + let positionY = 0; + + if (ctx) { + ctx.font = fontString; + setCanvasLetterSpacing({ ctx, letterSpacingPx: style.letterSpacing }); + content = wrapSubtitleText({ + ctx, + text: caption.text, + maxWidth, + }); + const measurement = measureWrappedTextBlock({ + ctx, + content, + canvasHeight: canvasSize.height, + textAlign: style.textAlign, + background: style.background, + fontSize: style.fontSize, + lineHeight: style.lineHeight, + }); + positionX = resolvePositionX({ + canvasWidth: canvasSize.width, + textAlign: style.textAlign, + placement: style.placement, + visualRect: measurement.visualRect, + }); + positionY = resolvePositionY({ + canvasHeight: canvasSize.height, + placement: style.placement, + visualRect: measurement.visualRect, + }); + } + + return { + ...DEFAULTS.text.element, + name: `Caption ${index + 1}`, + content, + duration: Math.round(caption.duration * TICKS_PER_SECOND), + startTime: Math.round(caption.startTime * TICKS_PER_SECOND), + fontSize: style.fontSize, + fontFamily: style.fontFamily, + color: style.color, + textAlign: style.textAlign, + fontWeight: style.fontWeight, + fontStyle: style.fontStyle, + textDecoration: style.textDecoration, + letterSpacing: style.letterSpacing, + lineHeight: style.lineHeight, + background: style.background, + transform: { + ...DEFAULTS.text.element.transform, + position: { + x: positionX, + y: positionY, + }, + }, + }; +} diff --git a/apps/web/src/lib/subtitles/insert.ts b/apps/web/src/lib/subtitles/insert.ts new file mode 100644 index 00000000..1bb04fc8 --- /dev/null +++ b/apps/web/src/lib/subtitles/insert.ts @@ -0,0 +1,40 @@ +import type { EditorCore } from "@/core"; +import { + AddTrackCommand, + BatchCommand, + InsertElementCommand, +} from "@/lib/commands"; +import { buildSubtitleTextElement } from "./build-subtitle-text-element"; +import type { SubtitleCue } from "./types"; + +export function insertCaptionChunksAsTextTrack({ + editor, + captions, +}: { + editor: EditorCore; + captions: SubtitleCue[]; +}): string | null { + if (captions.length === 0) { + return null; + } + + const addTrackCommand = new AddTrackCommand("text", 0); + const trackId = addTrackCommand.getTrackId(); + const canvasSize = editor.project.getActive().settings.canvasSize; + const insertCommands = captions.map( + (caption, index) => + new InsertElementCommand({ + placement: { mode: "explicit", trackId }, + element: buildSubtitleTextElement({ + index, + caption, + canvasSize, + }), + }), + ); + editor.command.execute({ + command: new BatchCommand([addTrackCommand, ...insertCommands]), + }); + + return trackId; +} diff --git a/apps/web/src/lib/subtitles/parse.ts b/apps/web/src/lib/subtitles/parse.ts new file mode 100644 index 00000000..7a2d2342 --- /dev/null +++ b/apps/web/src/lib/subtitles/parse.ts @@ -0,0 +1,28 @@ +import { parseAss } from "./ass"; +import { parseSrt } from "./srt"; +import type { ParseSubtitleResult } from "./types"; +export type { ParseSubtitleResult, SubtitleCue } from "./types"; + +export function parseSubtitleFile({ + fileName, + input, +}: { + fileName: string; + input: string; +}): ParseSubtitleResult { + const extension = getFileExtension({ fileName }); + + switch (extension) { + case "srt": + return parseSrt({ input }); + case "ass": + return parseAss({ input }); + default: + throw new Error("Unsupported subtitle format"); + } +} + +function getFileExtension({ fileName }: { fileName: string }): string { + const extension = fileName.split(".").pop(); + return extension?.toLowerCase() ?? ""; +} diff --git a/apps/web/src/lib/subtitles/srt.ts b/apps/web/src/lib/subtitles/srt.ts new file mode 100644 index 00000000..197e96f9 --- /dev/null +++ b/apps/web/src/lib/subtitles/srt.ts @@ -0,0 +1,98 @@ +import type { ParseSubtitleResult, SubtitleCue } from "./types"; + +const TIMESTAMP_SEPARATOR = /\s*-->\s*/; +const TIMESTAMP_PATTERN = + /^(\d{2}:\d{2}:\d{2}[,.]\d{1,3})\s*-->\s*(\d{2}:\d{2}:\d{2}[,.]\d{1,3})/; + +export function parseSrt({ input }: { input: string }): ParseSubtitleResult { + const normalized = input.replace(/\r\n?/g, "\n").trim(); + if (!normalized) { + return { + captions: [], + skippedCueCount: 0, + warnings: [], + }; + } + + const blocks = normalized.split(/\n{2,}/); + const cues: SubtitleCue[] = []; + let skippedCueCount = 0; + + for (const block of blocks) { + const lines = block + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + + if (lines.length < 2) { + skippedCueCount += 1; + continue; + } + + const timestampIndex = TIMESTAMP_SEPARATOR.test(lines[0]) ? 0 : 1; + const timestampLine = lines[timestampIndex]; + if (!timestampLine || !TIMESTAMP_PATTERN.test(timestampLine)) { + skippedCueCount += 1; + continue; + } + + const textLines = lines.slice(timestampIndex + 1); + const text = textLines.join("\n").trim(); + if (!text) { + skippedCueCount += 1; + continue; + } + + const [rawStart, rawEnd] = timestampLine.split(TIMESTAMP_SEPARATOR); + if (!rawStart || !rawEnd) { + skippedCueCount += 1; + continue; + } + + const startTime = parseSrtTimestamp({ input: rawStart }); + const endTime = parseSrtTimestamp({ input: rawEnd }); + const duration = endTime - startTime; + + if ( + !Number.isFinite(startTime) || + !Number.isFinite(endTime) || + duration <= 0 + ) { + skippedCueCount += 1; + continue; + } + + cues.push({ + text, + startTime, + duration, + }); + } + + return { + captions: cues, + skippedCueCount, + warnings: [], + }; +} + +function parseSrtTimestamp({ input }: { input: string }): number { + const normalized = input.trim().replace(",", "."); + const match = normalized.match(/^(\d{2}):(\d{2}):(\d{2})\.(\d{1,3})$/); + if (!match) { + return Number.NaN; + } + + const [, hours, minutes, seconds, milliseconds] = match; + const parsedHours = Number.parseInt(hours, 10); + const parsedMinutes = Number.parseInt(minutes, 10); + const parsedSeconds = Number.parseInt(seconds, 10); + const parsedMilliseconds = Number.parseInt(milliseconds.padEnd(3, "0"), 10); + + return ( + parsedHours * 3600 + + parsedMinutes * 60 + + parsedSeconds + + parsedMilliseconds / 1000 + ); +} diff --git a/apps/web/src/lib/subtitles/types.ts b/apps/web/src/lib/subtitles/types.ts new file mode 100644 index 00000000..86f6525d --- /dev/null +++ b/apps/web/src/lib/subtitles/types.ts @@ -0,0 +1,46 @@ +import type { TextBackground, TextElement } from "@/lib/timeline"; +import type { CaptionChunk } from "@/lib/transcription/types"; + +export interface SubtitlePlacementStyle { + verticalAlign?: "top" | "middle" | "bottom"; + marginLeftRatio?: number; + marginRightRatio?: number; + marginVerticalRatio?: number; +} + +export interface SubtitleStyleOverrides { + /** + * Font size in app units (same coordinate space as TextElement.fontSize). + * Use fontSizeRatioOfPlayHeight when the source coordinate space is unknown + * (e.g. ASS files, where font size is relative to the script's play resolution). + */ + fontSize?: number; + /** + * Font size expressed as a fraction of the reference canvas height. + * Set by the ASS parser so the builder can convert to app units without + * the parser needing to know about the app's coordinate system. + * Takes precedence over fontSize when both are present. + */ + fontSizeRatioOfPlayHeight?: number; + fontFamily?: string; + color?: string; + background?: Pick & + Partial>; + textAlign?: TextElement["textAlign"]; + fontWeight?: TextElement["fontWeight"]; + fontStyle?: TextElement["fontStyle"]; + textDecoration?: TextElement["textDecoration"]; + letterSpacing?: number; + lineHeight?: number; + placement?: SubtitlePlacementStyle; +} + +export interface SubtitleCue extends CaptionChunk { + style?: SubtitleStyleOverrides; +} + +export interface ParseSubtitleResult { + captions: SubtitleCue[]; + skippedCueCount: number; + warnings: string[]; +}