RomanNum3ral
/
OpenCut
mirror of https://github.com/OpenCut-app/OpenCut.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: transcription diagnostics

This commit is contained in:
Maze Winther 2026-04-14 23:34:57 +02:00
parent f726d95295
commit 91e1232112
12 changed files with 1190 additions and 1000 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

575
apps/web/src/app/globals.css
View File

@ -1,284 +1,291 @@
@import "tailwindcss"; @import "tailwindcss";
/* Custom variant for dark mode */ /* Custom variant for dark mode */
@custom-variant dark (&:where(.dark, .dark *)); @custom-variant dark (&:where(.dark, .dark *));
/* Plugins */ /* Plugins */
@plugin "@tailwindcss/typography"; @plugin "@tailwindcss/typography";
@plugin "tailwindcss-animate"; @plugin "tailwindcss-animate";
:root { :root {
--background: hsl(0, 0%, 100%); --background: hsl(0, 0%, 100%);
--foreground: hsl(0 0% 11%); --foreground: hsl(0 0% 11%);
--card: hsl(0, 0%, 100%); --card: hsl(0, 0%, 100%);
--card-foreground: hsl(0 0% 11%); --card-foreground: hsl(0 0% 11%);
--popover: hsl(0, 0%, 100%); --popover: hsl(0, 0%, 100%);
--popover-hover: hsl(0, 0%, 96%); --popover-hover: hsl(0, 0%, 96%);
--popover-foreground: hsl(0 0% 2%); --popover-foreground: hsl(0 0% 2%);
--primary: hsl(200, 90%, 52%); --primary: hsl(200, 90%, 52%);
--primary-foreground: hsl(0, 0%, 100%); --primary-foreground: hsl(0, 0%, 100%);
--secondary: hsl(204, 100%, 97%); --secondary: hsl(204, 100%, 97%);
--secondary-border: hsl(204, 100%, 94%); --secondary-border: hsl(204, 100%, 94%);
--secondary-foreground: hsl(200, 98%, 39%); --secondary-foreground: hsl(200, 98%, 39%);
--muted: hsl(0 0% 85.1%); --muted: hsl(0 0% 85.1%);
--muted-foreground: hsl(0 0% 50%); --muted-foreground: hsl(0 0% 50%);
--accent: hsl(0, 0%, 96%); --accent: hsl(0, 0%, 96%);
--accent-foreground: hsl(0 0% 2%); --accent-foreground: hsl(0 0% 2%);
--destructive: hsl(0, 83%, 50%); --destructive: hsl(0, 83%, 50%);
--destructive-foreground: hsl(0, 0%, 100%); --destructive-foreground: hsl(0, 0%, 100%);
--constructive: hsl(141, 71%, 48%); --constructive: hsl(141, 71%, 48%);
--constructive-foreground: hsl(0, 0%, 100%); --constructive-foreground: hsl(0, 0%, 100%);
--border: hsl(0 0% 91%); --caution: hsl(38, 92%, 50%);
--input: hsl(0, 0%, 100%); --caution-foreground: hsl(0, 0%, 10%);
--ring: hsl(0, 0%, 55%); --border: hsl(0 0% 91%);
--chart-1: hsl(220 70% 50%); --input: hsl(0, 0%, 100%);
--chart-2: hsl(160 60% 45%); --ring: hsl(0, 0%, 55%);
--chart-3: hsl(30 80% 55%); --chart-1: hsl(220 70% 50%);
--chart-4: hsl(280 65% 60%); --chart-2: hsl(160 60% 45%);
--chart-5: hsl(340 75% 55%); --chart-3: hsl(30 80% 55%);
--sidebar-background: hsl(0 0% 96.1%); --chart-4: hsl(280 65% 60%);
--sidebar-foreground: hsl(0 0% 2%); --chart-5: hsl(340 75% 55%);
--sidebar-primary: hsl(0 0% 2%); --sidebar-background: hsl(0 0% 96.1%);
--sidebar-primary-foreground: hsl(0 0% 91%); --sidebar-foreground: hsl(0 0% 2%);
--sidebar-accent: hsl(0 0% 85.1%); --sidebar-primary: hsl(0 0% 2%);
--sidebar-accent-foreground: hsl(0 0% 2%); --sidebar-primary-foreground: hsl(0 0% 91%);
--sidebar-border: hsl(0 0% 85.1%); --sidebar-accent: hsl(0 0% 85.1%);
--sidebar-ring: hsl(0 0% 16.9%); --sidebar-accent-foreground: hsl(0 0% 2%);
--sidebar: hsl(0 0% 98%); --sidebar-border: hsl(0 0% 85.1%);
} --sidebar-ring: hsl(0 0% 16.9%);
--sidebar: hsl(0 0% 98%);
.panel { }
--background: hsl(210, 20%, 98%);
--foreground: hsl(0 0% 13%); .panel {
--card: hsl(0, 0%, 98%); --background: hsl(210, 20%, 98%);
--card-foreground: hsl(0 0% 13%); --foreground: hsl(0 0% 13%);
--primary-foreground: hsl(0, 0%, 98%); --card: hsl(0, 0%, 98%);
--secondary: hsl(204, 100%, 95%); --card-foreground: hsl(0 0% 13%);
--secondary-border: hsl(204, 100%, 92%); --primary-foreground: hsl(0, 0%, 98%);
--secondary-foreground: hsl(200, 98%, 37%); --secondary: hsl(204, 100%, 95%);
--muted: hsl(0 0% 83.1%); --secondary-border: hsl(204, 100%, 92%);
--muted-foreground: hsl(0 0% 48%); --secondary-foreground: hsl(200, 98%, 37%);
--accent: hsl(0, 0%, 93%); --muted: hsl(0 0% 83.1%);
--accent-foreground: hsl(0 0% 5%); --muted-foreground: hsl(0 0% 48%);
--destructive-foreground: hsl(0, 0%, 98%); --accent: hsl(0, 0%, 93%);
--constructive-foreground: hsl(0, 0%, 98%); --accent-foreground: hsl(0 0% 5%);
--border: hsl(0 0% 87%); --destructive-foreground: hsl(0, 0%, 98%);
--input: hsl(0 0% 93%); --constructive-foreground: hsl(0, 0%, 98%);
--ring: hsl(0, 0%, 53%); --border: hsl(0 0% 87%);
} --input: hsl(0 0% 93%);
--ring: hsl(0, 0%, 53%);
.dark { }
--background: hsl(0, 0%, 5%);
--foreground: hsl(0 0% 87%); .dark {
--card: hsl(0, 0%, 5%); --background: hsl(0, 0%, 5%);
--card-foreground: hsl(0 0% 87%); --foreground: hsl(0 0% 87%);
--popover: hsl(0, 0%, 5%); --card: hsl(0, 0%, 5%);
--popover-hover: hsl(0, 0%, 13%); --card-foreground: hsl(0 0% 87%);
--popover-foreground: hsl(0 0% 95%); --popover: hsl(0, 0%, 5%);
--secondary: hsl(204, 100%, 12%); --popover-hover: hsl(0, 0%, 13%);
--secondary-border: hsl(204, 100%, 15%); --popover-foreground: hsl(0 0% 95%);
--secondary-foreground: hsl(200, 98%, 61%); --secondary: hsl(204, 100%, 12%);
--muted: hsl(0 0% 20%); --secondary-border: hsl(204, 100%, 15%);
--accent: hsl(0, 0%, 14%); --secondary-foreground: hsl(200, 98%, 61%);
--accent-foreground: hsl(0 0% 95%); --muted: hsl(0 0% 20%);
--border: hsl(0 0% 16%); --accent: hsl(0, 0%, 14%);
--input: hsl(0 0% 5%); --accent-foreground: hsl(0 0% 95%);
--ring: hsl(0, 0%, 50%); --border: hsl(0 0% 16%);
--sidebar-background: hsl(0 0% 8%); --input: hsl(0 0% 5%);
--sidebar-foreground: hsl(0 0% 95%); --ring: hsl(0, 0%, 50%);
--sidebar-primary: hsl(0 0% 95%); --caution: hsl(38, 92%, 60%);
--sidebar-primary-foreground: hsl(0 0% 15%); --caution-foreground: hsl(0, 0%, 10%);
--sidebar-accent: hsl(0 0% 20%); --sidebar-background: hsl(0 0% 8%);
--sidebar-accent-foreground: hsl(0 0% 95%); --sidebar-foreground: hsl(0 0% 95%);
--sidebar-border: hsl(0 0% 20%); --sidebar-primary: hsl(0 0% 95%);
--sidebar-ring: hsl(0 0% 83.1%); --sidebar-primary-foreground: hsl(0 0% 15%);
--sidebar: hsl(0 0% 6%); --sidebar-accent: hsl(0 0% 20%);
} --sidebar-accent-foreground: hsl(0 0% 95%);
--sidebar-border: hsl(0 0% 20%);
.dark .panel { --sidebar-ring: hsl(0 0% 83.1%);
--background: hsl(0 0% 10%); --sidebar: hsl(0 0% 6%);
--foreground: hsl(0 0% 85%); }
--card: hsl(0, 0%, 10%);
--card-foreground: hsl(0 0% 85%); .dark .panel {
--secondary: hsl(204, 67%, 9%); --background: hsl(0 0% 10%);
--secondary-border: hsl(204, 100%, 14%); --foreground: hsl(0 0% 85%);
--secondary-foreground: hsl(200, 98%, 63%); --card: hsl(0, 0%, 10%);
--muted: hsl(0 0% 22%); --card-foreground: hsl(0 0% 85%);
--accent: hsl(0, 0%, 15%); --secondary: hsl(204, 67%, 9%);
--accent-foreground: hsl(0 0% 93%); --secondary-border: hsl(204, 100%, 14%);
--border: hsl(0 0% 18%); --secondary-foreground: hsl(200, 98%, 63%);
--input: hsl(0 0% 22%); --muted: hsl(0 0% 22%);
--ring: hsl(0, 0%, 52%); --accent: hsl(0, 0%, 15%);
} --accent-foreground: hsl(0 0% 93%);
--border: hsl(0 0% 18%);
@layer base { --input: hsl(0 0% 22%);
/* --ring: hsl(0, 0%, 52%);
The default border color has changed to `currentcolor` in Tailwind CSS v4, }
so we've added these compatibility styles to make sure everything still
looks the same as it did with Tailwind CSS v3. @layer base {
/*
If we ever want to remove these styles, we need to add an explicit border The default border color has changed to `currentcolor` in Tailwind CSS v4,
color utility to any element that depends on these defaults. so we've added these compatibility styles to make sure everything still
*/ looks the same as it did with Tailwind CSS v3.
*,
::after, If we ever want to remove these styles, we need to add an explicit border
::before, color utility to any element that depends on these defaults.
::backdrop, */
::file-selector-button { *,
border-color: var(--color-gray-200, currentcolor); ::after,
} ::before,
/* Other default base styles */ ::backdrop,
* { ::file-selector-button {
@apply border-border; border-color: var(--color-gray-200, currentcolor);
} }
body { /* Other default base styles */
@apply bg-background text-foreground; * {
/* Prevent back/forward swipe */ @apply border-border;
overscroll-behavior-x: contain; }
} body {
::selection { @apply bg-background text-foreground;
@apply bg-primary/35 selection:text-primary-foreground; /* Prevent back/forward swipe */
} overscroll-behavior-x: contain;
} }
::selection {
@theme inline { @apply bg-primary/35 selection:text-primary-foreground;
/* Responsive breakpoints */ }
--breakpoint-xs: 30rem; }
/* Typography */ @theme inline {
--font-sans: var(--font-inter), sans-serif; /* Responsive breakpoints */
--breakpoint-xs: 30rem;
/* Font sizes */
--text-xs: 0.72rem; /* Typography */
--text-sm: 0.79rem; --font-sans: var(--font-inter), sans-serif;
--text-base: 0.92rem;
--text-base--line-height: calc(1.5 / 0.95); /* Font sizes */
--text-xs--line-height: calc(1 / 0.8); --text-xs: 0.72rem;
--text-sm: 0.79rem;
/* Border radius */ --text-base: 0.92rem;
--radius-lg: 0.82rem; --text-base--line-height: calc(1.5 / 0.95);
--radius-md: 0.65rem; --text-xs--line-height: calc(1 / 0.8);
--radius-sm: 0.35rem;
/* Border radius */
/* Palette mapped to root design tokens */ --radius-lg: 0.82rem;
--color-background: var(--background); --radius-md: 0.65rem;
--color-foreground: var(--foreground); --radius-sm: 0.35rem;
--color-card: var(--card); /* Palette mapped to root design tokens */
--color-card-foreground: var(--card-foreground); --color-background: var(--background);
--color-foreground: var(--foreground);
--color-popover: var(--popover);
--color-popover-hover: var(--popover-hover); --color-card: var(--card);
--color-popover-foreground: var(--popover-foreground); --color-card-foreground: var(--card-foreground);
--color-primary: var(--primary); --color-popover: var(--popover);
--color-primary-foreground: var(--primary-foreground); --color-popover-hover: var(--popover-hover);
--color-secondary: var(--secondary); --color-popover-foreground: var(--popover-foreground);
--color-secondary-border: var(--secondary-border);
--color-secondary-foreground: var(--secondary-foreground); --color-primary: var(--primary);
--color-primary-foreground: var(--primary-foreground);
--color-muted: var(--muted); --color-secondary: var(--secondary);
--color-muted-foreground: var(--muted-foreground); --color-secondary-border: var(--secondary-border);
--color-secondary-foreground: var(--secondary-foreground);
--color-accent: var(--accent);
--color-accent-foreground: var(--accent-foreground); --color-muted: var(--muted);
--color-muted-foreground: var(--muted-foreground);
--color-destructive: var(--destructive);
--color-destructive-foreground: var(--destructive-foreground); --color-accent: var(--accent);
--color-accent-foreground: var(--accent-foreground);
--color-constructive: var(--constructive);
--color-constructive-foreground: var(--constructive-foreground); --color-destructive: var(--destructive);
--color-destructive-foreground: var(--destructive-foreground);
--color-border: var(--border);
--color-input: var(--input); --color-constructive: var(--constructive);
--color-ring: var(--ring); --color-constructive-foreground: var(--constructive-foreground);
/* Chart colors */ --color-caution: var(--caution);
--color-chart-1: var(--chart-1); --color-caution-foreground: var(--caution-foreground);
--color-chart-2: var(--chart-2);
--color-chart-3: var(--chart-3); --color-border: var(--border);
--color-chart-4: var(--chart-4); --color-input: var(--input);
--color-chart-5: var(--chart-5); --color-ring: var(--ring);
/* Sidebar */ /* Chart colors */
--color-sidebar: var(--sidebar-background); --color-chart-1: var(--chart-1);
--color-sidebar-foreground: var(--sidebar-foreground); --color-chart-2: var(--chart-2);
--color-sidebar-primary: var(--sidebar-primary); --color-chart-3: var(--chart-3);
--color-sidebar-primary-foreground: var(--sidebar-primary-foreground); --color-chart-4: var(--chart-4);
--color-sidebar-accent: var(--sidebar-accent); --color-chart-5: var(--chart-5);
--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
--color-sidebar-border: var(--sidebar-border); /* Sidebar */
--color-sidebar-ring: var(--sidebar-ring); --color-sidebar: var(--sidebar-background);
--color-sidebar-foreground: var(--sidebar-foreground);
/* Animations */ --color-sidebar-primary: var(--sidebar-primary);
--animate-accordion-down: accordion-down 0.2s ease-out; --color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
--animate-accordion-up: accordion-up 0.2s ease-out; --color-sidebar-accent: var(--sidebar-accent);
--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
@keyframes accordion-down { --color-sidebar-border: var(--sidebar-border);
from { --color-sidebar-ring: var(--sidebar-ring);
height: 0;
} /* Animations */
to { --animate-accordion-down: accordion-down 0.2s ease-out;
height: var(--radix-accordion-content-height); --animate-accordion-up: accordion-up 0.2s ease-out;
}
} @keyframes accordion-down {
from {
@keyframes accordion-up { height: 0;
from { }
height: var(--radix-accordion-content-height); to {
} height: var(--radix-accordion-content-height);
to { }
height: 0; }
}
} @keyframes accordion-up {
} from {
height: var(--radix-accordion-content-height);
@utility scrollbar-hidden { }
-ms-overflow-style: none; to {
scrollbar-width: none; height: 0;
&::-webkit-scrollbar { }
display: none; }
} }
}
@utility scrollbar-hidden {
@utility scrollbar-x-hidden { -ms-overflow-style: none;
-ms-overflow-style: none; scrollbar-width: none;
scrollbar-width: none; &::-webkit-scrollbar {
&::-webkit-scrollbar:horizontal { display: none;
display: none; }
} }
}
@utility scrollbar-x-hidden {
@utility scrollbar-y-hidden { -ms-overflow-style: none;
-ms-overflow-style: none; scrollbar-width: none;
scrollbar-width: none; &::-webkit-scrollbar:horizontal {
&::-webkit-scrollbar:vertical { display: none;
display: none; }
} }
}
@utility scrollbar-y-hidden {
@utility scrollbar-thin { -ms-overflow-style: none;
&::-webkit-scrollbar { scrollbar-width: none;
width: 7px; &::-webkit-scrollbar:vertical {
height: 7px; display: none;
} }
&::-webkit-scrollbar-track { }
background: transparent;
} @utility scrollbar-thin {
&::-webkit-scrollbar-thumb { &::-webkit-scrollbar {
background: var(--border); width: 7px;
border-radius: 4px; height: 7px;
} }
&::-webkit-scrollbar-thumb:hover { &::-webkit-scrollbar-track {
background: var(--muted-foreground); background: transparent;
} }
} &::-webkit-scrollbar-thumb {
background: var(--border);
@layer base { border-radius: 4px;
* { }
@apply border-border outline-ring/50; &::-webkit-scrollbar-thumb:hover {
} background: var(--muted-foreground);
body { }
@apply bg-background text-foreground; }
}
} @layer base {
* {
@apply border-border outline-ring/50;
}
body {
@apply bg-background text-foreground;
}
}

99
apps/web/src/components/editor/panels/assets/views/captions.tsx
View File

@ -10,9 +10,8 @@ import {
import { useReducer, useRef, useState } from "react"; import { useReducer, useRef, useState } from "react";
import { extractTimelineAudio } from "@/lib/media/mediabunny"; import { extractTimelineAudio } from "@/lib/media/mediabunny";
import { useEditor } from "@/hooks/use-editor"; import { useEditor } from "@/hooks/use-editor";
import { import { TRANSCRIPTION_DIAGNOSTICS_SCOPE } from "@/lib/transcription/diagnostics";
DEFAULT_TRANSCRIPTION_SAMPLE_RATE, import { DEFAULT_TRANSCRIPTION_SAMPLE_RATE } from "@/lib/transcription/audio";
} from "@/lib/transcription/audio";
import { TRANSCRIPTION_LANGUAGES } from "@/lib/transcription/supported-languages"; import { TRANSCRIPTION_LANGUAGES } from "@/lib/transcription/supported-languages";
import type { import type {
CaptionChunk, CaptionChunk,
@ -31,8 +30,23 @@ import {
SectionField, SectionField,
SectionFields, SectionFields,
} from "@/components/section"; } from "@/components/section";
import { CloudUploadIcon } from "@hugeicons/core-free-icons"; import { AlertCircleIcon, CloudUploadIcon } from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react"; import { HugeiconsIcon } from "@hugeicons/react";
import {
Tooltip,
TooltipContent,
TooltipProvider,
TooltipTrigger,
} from "@/components/ui/tooltip";
import type { DiagnosticSeverity } from "@/lib/diagnostics/types";
const DIAGNOSTIC_BUTTON_VARIANT: Record<
DiagnosticSeverity,
"caution" | "destructive-foreground"
> = {
caution: "caution",
error: "destructive-foreground",
};
type ProcessingState = type ProcessingState =
| { status: "idle"; error: string | null; warnings: string[] } | { status: "idle"; error: string | null; warnings: string[] }
@ -44,7 +58,11 @@ type ProcessingAction =
| { type: "succeed"; warnings: string[] } | { type: "succeed"; warnings: string[] }
| { type: "fail"; error: string }; | { type: "fail"; error: string };
const IDLE_STATE: ProcessingState = { status: "idle", error: null, warnings: [] }; const IDLE_STATE: ProcessingState = {
status: "idle",
error: null,
warnings: [],
};
function processingReducer( function processingReducer(
state: ProcessingState, state: ProcessingState,
@ -73,6 +91,10 @@ export function Captions() {
const isProcessing = processing.status === "processing"; const isProcessing = processing.status === "processing";
const activeDiagnostics = useEditor((e) =>
e.diagnostics.getActive({ scope: TRANSCRIPTION_DIAGNOSTICS_SCOPE }),
);
const handleProgress = (progress: TranscriptionProgress) => { const handleProgress = (progress: TranscriptionProgress) => {
if (progress.status === "loading-model") { if (progress.status === "loading-model") {
dispatch({ dispatch({
@ -84,7 +106,11 @@ export function Captions() {
} }
}; };
const insertCaptions = ({ captions }: { captions: CaptionChunk[] }): boolean => { const insertCaptions = ({
captions,
}: {
captions: CaptionChunk[];
}): boolean => {
const trackId = insertCaptionChunksAsTextTrack({ editor, captions }); const trackId = insertCaptionChunksAsTextTrack({ editor, captions });
return trackId !== null; return trackId !== null;
}; };
@ -123,7 +149,10 @@ export function Captions() {
console.error("Transcription failed:", error); console.error("Transcription failed:", error);
dispatch({ dispatch({
type: "fail", type: "fail",
error: error instanceof Error ? error.message : "An unexpected error occurred", error:
error instanceof Error
? error.message
: "An unexpected error occurred",
}); });
} }
}; };
@ -168,7 +197,10 @@ export function Captions() {
console.error("Subtitle import failed:", error); console.error("Subtitle import failed:", error);
dispatch({ dispatch({
type: "fail", type: "fail",
error: error instanceof Error ? error.message : "An unexpected error occurred", error:
error instanceof Error
? error.message
: "An unexpected error occurred",
}); });
} }
}; };
@ -208,17 +240,38 @@ export function Captions() {
title="Captions" title="Captions"
contentClassName="px-0 flex flex-col h-full" contentClassName="px-0 flex flex-col h-full"
actions={ actions={
<Button <TooltipProvider>
type="button" <div className="flex items-center gap-1.5">
variant="outline" {!isProcessing &&
size="sm" activeDiagnostics.map((diagnostic) => (
onClick={handleImportClick} <Tooltip key={diagnostic.id}>
disabled={isProcessing} <TooltipTrigger asChild>
className="items-center justify-center gap-1.5" <Button
> variant={DIAGNOSTIC_BUTTON_VARIANT[diagnostic.severity]}
<HugeiconsIcon icon={CloudUploadIcon} /> size="icon"
Import aria-label={diagnostic.message}
</Button> >
<HugeiconsIcon icon={AlertCircleIcon} size={16} />
</Button>
</TooltipTrigger>
<TooltipContent>
{diagnostic.message}
</TooltipContent>
</Tooltip>
))}
<Button
type="button"
variant="outline"
size="sm"
onClick={handleImportClick}
disabled={isProcessing}
className="items-center justify-center gap-1.5"
>
<HugeiconsIcon icon={CloudUploadIcon} />
Import
</Button>
</div>
</TooltipProvider>
} }
ref={containerRef} ref={containerRef}
> >
@ -229,7 +282,11 @@ export function Captions() {
className="hidden" className="hidden"
onChange={(event) => void handleFileChange({ event })} onChange={(event) => void handleFileChange({ event })}
/> />
<Section showTopBorder={false} showBottomBorder={false} className="flex-1"> <Section
showTopBorder={false}
showBottomBorder={false}
className="flex-1"
>
<SectionContent className="flex flex-col gap-4 h-full pt-1"> <SectionContent className="flex flex-col gap-4 h-full pt-1">
<SectionFields> <SectionFields>
<SectionField label="Language"> <SectionField label="Language">
@ -256,7 +313,7 @@ export function Captions() {
type="button" type="button"
className="mt-auto w-full" className="mt-auto w-full"
onClick={handleGenerateTranscript} onClick={handleGenerateTranscript}
disabled={isProcessing} disabled={isProcessing || activeDiagnostics.length > 0}
> >
{isProcessing && <Spinner className="mr-1" />} {isProcessing && <Spinner className="mr-1" />}
{isProcessing ? processing.step : "Generate transcript"} {isProcessing ? processing.step : "Generate transcript"}

910
apps/web/src/components/editor/panels/assets/views/stickers.tsx
View File

@ -1,455 +1,455 @@
"use client"; "use client";
import Image from "next/image"; import Image from "next/image";
import type { CSSProperties } from "react"; import type { CSSProperties } from "react";
import { useEffect, useState } from "react"; import { useEffect, useState } from "react";
import { toast } from "sonner"; import { toast } from "sonner";
import { DraggableItem } from "@/components/editor/panels/assets/draggable-item"; import { DraggableItem } from "@/components/editor/panels/assets/draggable-item";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input"; import { Input } from "@/components/ui/input";
import { Spinner } from "@/components/ui/spinner"; import { Spinner } from "@/components/ui/spinner";
import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { useEditor } from "@/hooks/use-editor"; import { useEditor } from "@/hooks/use-editor";
import { resolveStickerIntrinsicSize } from "@/lib/stickers"; import { resolveStickerIntrinsicSize } from "@/lib/stickers";
import { import {
buildGraphicElement, buildGraphicElement,
buildStickerElement, buildStickerElement,
} from "@/lib/timeline/element-utils"; } from "@/lib/timeline/element-utils";
import { STICKER_CATEGORIES } from "@/lib/stickers/categories"; import { STICKER_CATEGORIES } from "@/lib/stickers/categories";
import { getRegionLabel, resolveQueryToRegions } from "@/lib/stickers"; import { getRegionLabel, resolveQueryToRegions } from "@/lib/stickers";
import { parseShapeStickerId } from "@/lib/stickers/providers/shapes"; import { parseShapeStickerId } from "@/lib/stickers/providers/shapes";
import type { TimelineDragData } from "@/lib/timeline/drag"; import type { TimelineDragData } from "@/lib/timeline/drag";
import type { import type {
StickerBrowseSection, StickerBrowseSection,
StickerCategory, StickerCategory,
StickerItem as StickerData, StickerItem as StickerData,
} from "@/lib/stickers"; } from "@/lib/stickers";
import { useStickersStore } from "@/stores/stickers-store"; import { useStickersStore } from "@/stores/stickers-store";
import { cn } from "@/utils/ui"; import { cn } from "@/utils/ui";
import { import {
HappyIcon, HappyIcon,
} from "@hugeicons/core-free-icons"; } from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react"; import { HugeiconsIcon } from "@hugeicons/react";
export function StickersView() { export function StickersView() {
const { const {
browseContent, browseContent,
browseStickers, browseStickers,
searchQuery, searchQuery,
searchStickers, searchStickers,
selectedCategory, selectedCategory,
setSearchQuery, setSearchQuery,
setSelectedCategory, setSelectedCategory,
viewMode, viewMode,
} = useStickersStore(); } = useStickersStore();
useEffect(() => { useEffect(() => {
if (viewMode === "browse" && !browseContent) { if (viewMode === "browse" && !browseContent) {
void browseStickers(); void browseStickers();
} }
}, [browseContent, browseStickers, viewMode]); }, [browseContent, browseStickers, viewMode]);
return ( return (
<div className="flex h-full flex-col py-2"> <div className="flex h-full flex-col py-2">
<div className="px-2"> <div className="px-2">
<Input <Input
size="sm" size="sm"
variant="default" variant="default"
placeholder="Search..." placeholder="Search..."
value={searchQuery} value={searchQuery}
onChange={(e) => { onChange={(e) => {
setSearchQuery({ query: e.target.value }); setSearchQuery({ query: e.target.value });
void searchStickers({ query: e.target.value }); void searchStickers({ query: e.target.value });
}} }}
showClearIcon showClearIcon
onClear={() => { onClear={() => {
setSearchQuery({ query: "" }); setSearchQuery({ query: "" });
void searchStickers({ query: "" }); void searchStickers({ query: "" });
}} }}
className="w-full" className="w-full"
containerClassName="w-full" containerClassName="w-full"
/> />
</div> </div>
<Tabs <Tabs
value={selectedCategory} value={selectedCategory}
onValueChange={(value) => { onValueChange={(value) => {
setSelectedCategory({ category: value as StickerCategory }); setSelectedCategory({ category: value as StickerCategory });
}} }}
variant="underline" variant="underline"
className="mt-2 flex min-h-0 flex-1 flex-col" className="mt-2 flex min-h-0 flex-1 flex-col"
> >
<TabsList aria-label="Sticker categories"> <TabsList aria-label="Sticker categories">
{Object.entries(STICKER_CATEGORIES).map(([key, label]) => ( {Object.entries(STICKER_CATEGORIES).map(([key, label]) => (
<TabsTrigger key={key} value={key}> <TabsTrigger key={key} value={key}>
{label} {label}
</TabsTrigger> </TabsTrigger>
))} ))}
</TabsList> </TabsList>
<div className="min-h-0 flex-1 overflow-y-auto px-4 pt-4"> <div className="min-h-0 flex-1 overflow-y-auto px-4 pt-4">
<StickersContentView /> <StickersContentView />
</div> </div>
</Tabs> </Tabs>
</div> </div>
); );
} }
function StickerGrid({ function StickerGrid({
items, items,
shouldCapSize = false, shouldCapSize = false,
}: { }: {
items: StickerData[]; items: StickerData[];
shouldCapSize?: boolean; shouldCapSize?: boolean;
}) { }) {
const gridStyle: CSSProperties & { const gridStyle: CSSProperties & {
"--sticker-min": string; "--sticker-min": string;
"--sticker-max"?: string; "--sticker-max"?: string;
} = { } = {
gridTemplateColumns: shouldCapSize gridTemplateColumns: shouldCapSize
? "repeat(auto-fill, minmax(var(--sticker-min, 80px), var(--sticker-max, 140px)))" ? "repeat(auto-fill, minmax(var(--sticker-min, 80px), var(--sticker-max, 140px)))"
: "repeat(auto-fill, minmax(var(--sticker-min, 80px), 1fr))", : "repeat(auto-fill, minmax(var(--sticker-min, 80px), 1fr))",
"--sticker-min": "80px", "--sticker-min": "80px",
...(shouldCapSize ? { "--sticker-max": "140px" } : {}), ...(shouldCapSize ? { "--sticker-max": "140px" } : {}),
}; };
return ( return (
<div className="grid gap-2" style={gridStyle}> <div className="grid gap-2" style={gridStyle}>
{items.map((item) => ( {items.map((item) => (
<StickerItem key={item.id} item={item} shouldCapSize={shouldCapSize} /> <StickerItem key={item.id} item={item} shouldCapSize={shouldCapSize} />
))} ))}
</div> </div>
); );
} }
function StickerRow({ items }: { items: StickerData[] }) { function StickerRow({ items }: { items: StickerData[] }) {
return ( return (
<div className="flex gap-2 overflow-x-auto pb-1 scrollbar-hidden"> <div className="flex gap-2 overflow-x-auto pb-1 scrollbar-hidden">
{items.map((item) => ( {items.map((item) => (
<div key={item.id} className="w-20 shrink-0"> <div key={item.id} className="w-20 shrink-0">
<StickerItem item={item} shouldCapSize containerClassName="w-full" /> <StickerItem item={item} shouldCapSize containerClassName="w-full" />
</div> </div>
))} ))}
</div> </div>
); );
} }
function EmptyView({ message }: { message: string }) { function EmptyView({ message }: { message: string }) {
return ( return (
<div className="bg-background flex h-full flex-col items-center justify-center gap-3 p-4"> <div className="bg-background flex h-full flex-col items-center justify-center gap-3 p-4">
<HugeiconsIcon <HugeiconsIcon
icon={HappyIcon} icon={HappyIcon}
className="text-muted-foreground size-10" className="text-muted-foreground size-10"
/> />
<div className="flex flex-col gap-2 text-center"> <div className="flex flex-col gap-2 text-center">
<p className="text-lg font-medium">No stickers found</p> <p className="text-lg font-medium">No stickers found</p>
<p className="text-muted-foreground text-sm text-balance">{message}</p> <p className="text-muted-foreground text-sm text-balance">{message}</p>
</div> </div>
</div> </div>
); );
} }
function RegionBanner({ region }: { region: string }) { function RegionBanner({ region }: { region: string }) {
return ( return (
<div className="flex h-7 items-center gap-1.5 rounded-lg border border-sky-100 bg-sky-50 px-2"> <div className="flex h-7 items-center gap-1.5 rounded-lg border border-sky-100 bg-sky-50 px-2">
<svg <svg
width="12" width="12"
height="12" height="12"
viewBox="0 0 24 24" viewBox="0 0 24 24"
fill="none" fill="none"
stroke="currentColor" stroke="currentColor"
strokeWidth="2" strokeWidth="2"
strokeLinecap="round" strokeLinecap="round"
strokeLinejoin="round" strokeLinejoin="round"
className="shrink-0 text-sky-600" className="shrink-0 text-sky-600"
aria-hidden="true" aria-hidden="true"
> >
<path d="M20 10c0 6-8 12-8 12s-8-6-8-12a8 8 0 0 1 16 0Z" /> <path d="M20 10c0 6-8 12-8 12s-8-6-8-12a8 8 0 0 1 16 0Z" />
<circle cx="12" cy="10" r="3" /> <circle cx="12" cy="10" r="3" />
</svg> </svg>
<span className="text-xs font-semibold text-sky-600">{region}</span> <span className="text-xs font-semibold text-sky-600">{region}</span>
</div> </div>
); );
} }
function StickersContentView() { function StickersContentView() {
const { const {
browseContent, browseContent,
clearRecentStickers, clearRecentStickers,
isBrowsing, isBrowsing,
isSearching, isSearching,
searchQuery, searchQuery,
searchResults, searchResults,
selectedCategory, selectedCategory,
setSelectedCategory, setSelectedCategory,
viewMode, viewMode,
} = useStickersStore(); } = useStickersStore();
if (viewMode === "search") { if (viewMode === "search") {
if (isSearching) { if (isSearching) {
return ( return (
<div className="flex items-center justify-center py-8"> <div className="flex items-center justify-center py-8">
<Spinner className="text-muted-foreground size-6" /> <Spinner className="text-muted-foreground size-6" />
</div> </div>
); );
} }
if (searchResults?.items.length) { if (searchResults?.items.length) {
const normalizedQuery = searchQuery.trim().toLowerCase(); const normalizedQuery = searchQuery.trim().toLowerCase();
const isRegionSearch = const isRegionSearch =
selectedCategory === "flags" && selectedCategory === "flags" &&
resolveQueryToRegions({ query: normalizedQuery }) !== null; resolveQueryToRegions({ query: normalizedQuery }) !== null;
const regionLabel = getRegionLabel({ query: normalizedQuery }); const regionLabel = getRegionLabel({ query: normalizedQuery });
return ( return (
<div className="flex flex-col gap-3 pb-4"> <div className="flex flex-col gap-3 pb-4">
{isRegionSearch && <RegionBanner region={regionLabel} />} {isRegionSearch && <RegionBanner region={regionLabel} />}
<div className="flex items-center justify-between"> <div className="flex items-center justify-between">
<span className="text-muted-foreground text-sm"> <span className="text-muted-foreground text-sm">
{searchResults.total} results {searchResults.total} results
</span> </span>
</div> </div>
<StickerGrid items={searchResults.items} /> <StickerGrid items={searchResults.items} />
</div> </div>
); );
} }
// "all" tab search — sections are in browseContent, fall through to section rendering below // "all" tab search — sections are in browseContent, fall through to section rendering below
if (selectedCategory !== "all" && searchQuery) { if (selectedCategory !== "all" && searchQuery) {
return <EmptyView message={`No stickers found for "${searchQuery}"`} />; return <EmptyView message={`No stickers found for "${searchQuery}"`} />;
} }
} }
if (isBrowsing && !browseContent) { if (isBrowsing && !browseContent) {
return ( return (
<div className="flex items-center justify-center py-8"> <div className="flex items-center justify-center py-8">
<Spinner className="text-muted-foreground size-6" /> <Spinner className="text-muted-foreground size-6" />
</div> </div>
); );
} }
if (!browseContent?.sections.length) { if (!browseContent?.sections.length) {
const categoryLabel = STICKER_CATEGORIES[selectedCategory]; const categoryLabel = STICKER_CATEGORIES[selectedCategory];
return ( return (
<EmptyView <EmptyView
message={ message={
viewMode === "search" viewMode === "search"
? `No stickers found for "${searchQuery}"` ? `No stickers found for "${searchQuery}"`
: selectedCategory === "all" : selectedCategory === "all"
? "No stickers available yet." ? "No stickers available yet."
: `No stickers available in ${categoryLabel.toLowerCase()} yet.` : `No stickers available in ${categoryLabel.toLowerCase()} yet.`
} }
/> />
); );
} }
return ( return (
<div className="flex flex-col gap-4 pb-4"> <div className="flex flex-col gap-4 pb-4">
{browseContent.sections.map((section) => ( {browseContent.sections.map((section) => (
<StickerSection <StickerSection
key={section.id} key={section.id}
section={section} section={section}
onClearRecent={clearRecentStickers} onClearRecent={clearRecentStickers}
onSeeAll={(category) => { onSeeAll={(category) => {
setSelectedCategory({ category }); setSelectedCategory({ category });
}} }}
/> />
))} ))}
</div> </div>
); );
} }
function StickerSection({ function StickerSection({
section, section,
onClearRecent, onClearRecent,
onSeeAll, onSeeAll,
}: { }: {
section: StickerBrowseSection; section: StickerBrowseSection;
onClearRecent: () => void; onClearRecent: () => void;
onSeeAll: (category: StickerCategory) => void; onSeeAll: (category: StickerCategory) => void;
}) { }) {
const hasHeader = const hasHeader =
Boolean(section.title) || section.id === "recent" || section.action; Boolean(section.title) || section.id === "recent" || section.action;
return ( return (
<div className="flex flex-col gap-2"> <div className="flex flex-col gap-2">
{hasHeader && ( {hasHeader && (
<div className="flex items-center justify-between gap-2"> <div className="flex items-center justify-between gap-2">
{section.title ? ( {section.title ? (
<p className="text-xs text-muted-foreground">{section.title}</p> <p className="text-xs text-muted-foreground">{section.title}</p>
) : ( ) : (
<div /> <div />
)} )}
<div className="ml-auto flex items-center gap-2"> <div className="ml-auto flex items-center gap-2">
{section.id === "recent" && ( {section.id === "recent" && (
<Button <Button
onClick={onClearRecent} onClick={onClearRecent}
variant="text" variant="text"
size="sm" size="sm"
className="h-auto gap-1 p-0 text-xs text-muted-foreground" className="h-auto gap-1 p-0 text-xs text-muted-foreground"
> >
Clear Clear
</Button> </Button>
)} )}
{section.action?.type === "see-all" && section.action.category && ( {section.action?.type === "see-all" && section.action.category && (
<Button <Button
variant="text" variant="text"
size="sm" size="sm"
className="h-auto gap-1 p-0 text-xs text-primary" className="h-auto gap-1 p-0 text-xs text-primary"
onClick={() => { onClick={() => {
onSeeAll(section.action?.category as StickerCategory); onSeeAll(section.action?.category as StickerCategory);
}} }}
> >
See all See all
</Button> </Button>
)} )}
</div> </div>
</div> </div>
)} )}
{section.layout === "row" ? ( {section.layout === "row" ? (
<StickerRow items={section.items} /> <StickerRow items={section.items} />
) : ( ) : (
<StickerGrid items={section.items} /> <StickerGrid items={section.items} />
)} )}
</div> </div>
); );
} }
interface StickerItemProps { interface StickerItemProps {
item: StickerData; item: StickerData;
shouldCapSize?: boolean; shouldCapSize?: boolean;
containerClassName?: string; containerClassName?: string;
} }
function StickerItem({ function StickerItem({
item, item,
shouldCapSize = false, shouldCapSize = false,
containerClassName, containerClassName,
}: StickerItemProps) { }: StickerItemProps) {
const editor = useEditor(); const editor = useEditor();
const { addToRecentStickers } = useStickersStore(); const { addToRecentStickers } = useStickersStore();
const [isAdding, setIsAdding] = useState(false); const [isAdding, setIsAdding] = useState(false);
const [hasImageError, setHasImageError] = useState(false); const [hasImageError, setHasImageError] = useState(false);
useEffect(() => { useEffect(() => {
if (!item.id) { if (!item.id) {
return; return;
} }
setHasImageError(false); setHasImageError(false);
}, [item.id]); }, [item.id]);
const displayName = item.name; const displayName = item.name;
const shapePreset = const shapePreset =
item.provider === "shapes" ? parseShapeStickerId({ stickerId: item.id }) : null; item.provider === "shapes" ? parseShapeStickerId({ stickerId: item.id }) : null;
const handleAdd = async () => { const handleAdd = async () => {
setIsAdding(true); setIsAdding(true);
try { try {
const currentTime = editor.playback.getCurrentTime(); const currentTime = editor.playback.getCurrentTime();
let element: let element:
| ReturnType<typeof buildGraphicElement> | ReturnType<typeof buildGraphicElement>
| ReturnType<typeof buildStickerElement>; | ReturnType<typeof buildStickerElement>;
if (shapePreset) { if (shapePreset) {
element = buildGraphicElement({ element = buildGraphicElement({
definitionId: shapePreset.definitionId, definitionId: shapePreset.definitionId,
name: shapePreset.name, name: shapePreset.name,
startTime: currentTime, startTime: currentTime,
params: shapePreset.params, params: shapePreset.params,
}); });
} else { } else {
const { width: intrinsicWidth, height: intrinsicHeight } = const { width: intrinsicWidth, height: intrinsicHeight } =
await resolveStickerIntrinsicSize({ stickerId: item.id }); await resolveStickerIntrinsicSize({ stickerId: item.id });
element = buildStickerElement({ element = buildStickerElement({
stickerId: item.id, stickerId: item.id,
name: item.name, name: item.name,
startTime: currentTime, startTime: currentTime,
intrinsicWidth, intrinsicWidth,
intrinsicHeight, intrinsicHeight,
}); });
} }
editor.timeline.insertElement({ editor.timeline.insertElement({
placement: { mode: "auto" }, placement: { mode: "auto" },
element, element,
}); });
addToRecentStickers({ stickerId: item.id }); addToRecentStickers({ stickerId: item.id });
} catch (error) { } catch (error) {
console.error("Failed to add sticker:", error); console.error("Failed to add sticker:", error);
toast.error("Failed to add sticker to timeline"); toast.error("Failed to add sticker to timeline");
} finally { } finally {
setIsAdding(false); setIsAdding(false);
} }
}; };
const preview = ( const preview = (
<div className="flex size-full items-center justify-center p-3"> <div className="flex size-full items-center justify-center p-3">
{hasImageError ? ( {hasImageError ? (
<span className="text-muted-foreground text-center text-xs break-all"> <span className="text-muted-foreground text-center text-xs break-all">
{displayName} {displayName}
</span> </span>
) : ( ) : (
<Image <Image
src={item.previewUrl} src={item.previewUrl}
alt={displayName} alt={displayName}
width={64} width={64}
height={64} height={64}
className="size-full object-contain" className="size-full object-contain"
style={ style={
shouldCapSize shouldCapSize
? { ? {
maxWidth: "var(--sticker-max, 160px)", maxWidth: "var(--sticker-max, 160px)",
maxHeight: "var(--sticker-max, 160px)", maxHeight: "var(--sticker-max, 160px)",
} }
: undefined : undefined
} }
onError={() => setHasImageError(true)} onError={() => setHasImageError(true)}
loading="lazy" loading="lazy"
unoptimized unoptimized
/> />
)} )}
</div> </div>
); );
const dragData: TimelineDragData = shapePreset const dragData: TimelineDragData = shapePreset
? { ? {
id: item.id, id: item.id,
type: "graphic", type: "graphic",
name: displayName, name: displayName,
definitionId: shapePreset.definitionId, definitionId: shapePreset.definitionId,
params: shapePreset.params ?? {}, params: shapePreset.params ?? {},
} }
: { : {
id: item.id, id: item.id,
type: "sticker", type: "sticker",
name: displayName, name: displayName,
stickerId: item.id, stickerId: item.id,
}; };
return ( return (
<div <div
className={cn("relative", isAdding && "pointer-events-none opacity-50")} className={cn("relative", isAdding && "pointer-events-none opacity-50")}
> >
<DraggableItem <DraggableItem
name={displayName} name={displayName}
preview={preview} preview={preview}
dragData={dragData} dragData={dragData}
onAddToTimeline={handleAdd} onAddToTimeline={handleAdd}
aspectRatio={1} aspectRatio={1}
shouldShowLabel={false} shouldShowLabel={false}
isRounded isRounded
variant="card" variant="card"
containerClassName={containerClassName ?? "w-full"} containerClassName={containerClassName ?? "w-full"}
/> />
{isAdding && ( {isAdding && (
<div className="absolute inset-0 z-10 flex items-center justify-center rounded-md bg-black/60"> <div className="absolute inset-0 z-10 flex items-center justify-center rounded-md bg-black/60">
<Spinner className="size-6 text-white" /> <Spinner className="size-6 text-white" />
</div> </div>
)} )}
</div> </div>
); );
} }

1
apps/web/src/components/ui/button.tsx
View File

@ -14,6 +14,7 @@ const buttonVariants = cva(
"bg-destructive text-destructive-foreground hover:bg-destructive/80", "bg-destructive text-destructive-foreground hover:bg-destructive/80",
"destructive-foreground": "destructive-foreground":
"border bg-background hover:bg-destructive/15 text-destructive", "border bg-background hover:bg-destructive/15 text-destructive",
caution: "text-caution hover:bg-caution/10",
outline: "border border-border bg-background hover:bg-accent", outline: "border border-border bg-background hover:bg-accent",
secondary: secondary:
"bg-secondary text-secondary-foreground border border-secondary-border", "bg-secondary text-secondary-foreground border border-secondary-border",

5
apps/web/src/core/index.ts
View File

@ -9,8 +9,10 @@ import { SaveManager } from "./managers/save-manager";
import { AudioManager } from "./managers/audio-manager"; import { AudioManager } from "./managers/audio-manager";
import { SelectionManager } from "./managers/selection-manager"; import { SelectionManager } from "./managers/selection-manager";
import { ClipboardManager } from "./managers/clipboard-manager"; import { ClipboardManager } from "./managers/clipboard-manager";
import { DiagnosticsManager } from "./managers/diagnostics-manager";
import { registerDefaultEffects } from "@/lib/effects"; import { registerDefaultEffects } from "@/lib/effects";
import { registerDefaultMasks } from "@/lib/masks"; import { registerDefaultMasks } from "@/lib/masks";
import { registerTranscriptionDiagnostics } from "@/lib/transcription/diagnostics";
export class EditorCore { export class EditorCore {
private static instance: EditorCore | null = null; private static instance: EditorCore | null = null;
@ -25,6 +27,7 @@ export class EditorCore {
public readonly audio: AudioManager; public readonly audio: AudioManager;
public readonly selection: SelectionManager; public readonly selection: SelectionManager;
public readonly clipboard: ClipboardManager; public readonly clipboard: ClipboardManager;
public readonly diagnostics: DiagnosticsManager;
private constructor() { private constructor() {
registerDefaultEffects(); registerDefaultEffects();
@ -40,6 +43,8 @@ export class EditorCore {
this.audio = new AudioManager(this); this.audio = new AudioManager(this);
this.selection = new SelectionManager(this); this.selection = new SelectionManager(this);
this.clipboard = new ClipboardManager(this); this.clipboard = new ClipboardManager(this);
this.diagnostics = new DiagnosticsManager(this);
registerTranscriptionDiagnostics({ diagnostics: this.diagnostics });
this.playback.bindTimelineScope(); this.playback.bindTimelineScope();
this.command.registerReactor(() => { this.command.registerReactor(() => {
const activeScene = this.scenes.getActiveSceneOrNull(); const activeScene = this.scenes.getActiveSceneOrNull();

38
apps/web/src/core/managers/diagnostics-manager.ts Normal file
View File

@ -0,0 +1,38 @@
import type { EditorCore } from "@/core";
import type { DiagnosticDefinition } from "@/lib/diagnostics/types";
interface DiagnosticRegistration extends DiagnosticDefinition {
check: (editor: EditorCore) => boolean;
}
export class DiagnosticsManager {
private readonly registrations: DiagnosticRegistration[] = [];
private readonly listeners = new Set<() => void>();
constructor(private editor: EditorCore) {}
register(registration: DiagnosticRegistration): void {
this.registrations.push(registration);
this.notify();
}
getActive(options?: { scope?: string }): ReadonlyArray<DiagnosticDefinition> {
const candidates =
options?.scope !== undefined
? this.registrations.filter((r) => r.scope === options.scope)
: this.registrations;
return candidates.filter((r) => r.check(this.editor));
}
subscribe(listener: () => void): () => void {
this.listeners.add(listener);
return () => this.listeners.delete(listener);
}
notify(): void {
this.listeners.forEach((listener) => {
listener();
});
}
}

1
apps/web/src/hooks/use-editor.ts
View File

@ -32,6 +32,7 @@ export function useEditor<T>(
editor.renderer.subscribe(onChange), editor.renderer.subscribe(onChange),
editor.selection.subscribe(onChange), editor.selection.subscribe(onChange),
editor.clipboard.subscribe(onChange), editor.clipboard.subscribe(onChange),
editor.diagnostics.subscribe(onChange),
]; ];
return () => { return () => {
unsubscribers.forEach((unsubscribe) => { unsubscribers.forEach((unsubscribe) => {

8
apps/web/src/lib/diagnostics/types.ts Normal file
View File

@ -0,0 +1,8 @@
export type DiagnosticSeverity = "caution" | "error";
export interface DiagnosticDefinition {
id: string;
scope: string;
severity: DiagnosticSeverity;
message: string;
}

188
apps/web/src/lib/media/audio.ts
View File

@ -1,5 +1,6 @@
import type { import type {
AudioElement, AudioElement,
VideoElement,
LibraryAudioElement, LibraryAudioElement,
RetimeConfig, RetimeConfig,
SceneTracks, SceneTracks,
@ -12,9 +13,7 @@ import {
hasAnimatedVolume, hasAnimatedVolume,
resolveEffectiveAudioGain, resolveEffectiveAudioGain,
} from "@/lib/timeline/audio-state"; } from "@/lib/timeline/audio-state";
import { import { doesElementHaveEnabledAudio } from "@/lib/timeline/audio-separation";
doesElementHaveEnabledAudio,
} from "@/lib/timeline/audio-separation";
import { canElementHaveAudio, hasMediaId } from "@/lib/timeline/element-utils"; import { canElementHaveAudio, hasMediaId } from "@/lib/timeline/element-utils";
import { canTrackHaveAudio } from "@/lib/timeline"; import { canTrackHaveAudio } from "@/lib/timeline";
import { mediaSupportsAudio } from "@/lib/media/media-utils"; import { mediaSupportsAudio } from "@/lib/media/media-utils";
@ -83,22 +82,23 @@ export async function decodeAudioToFloat32({
return { samples, sampleRate: audioBuffer.sampleRate }; return { samples, sampleRate: audioBuffer.sampleRate };
} }
export async function collectAudioElements({ export interface AudibleElementCandidate {
element: AudioElement | VideoElement;
mediaAsset: MediaAsset | null;
}
export function collectAudibleCandidates({
tracks, tracks,
mediaAssets, mediaAssets,
audioContext,
}: { }: {
tracks: SceneTracks; tracks: SceneTracks;
mediaAssets: MediaAsset[]; mediaAssets: MediaAsset[];
audioContext: AudioContext; }): AudibleElementCandidate[] {
}): Promise<CollectedAudioElement[]> { const allTracks = [...tracks.overlay, tracks.main, ...tracks.audio];
const orderedTracks = [...tracks.overlay, tracks.main, ...tracks.audio]; const mediaMap = new Map(mediaAssets.map((a) => [a.id, a]));
const mediaMap = new Map<string, MediaAsset>( const candidates: AudibleElementCandidate[] = [];
mediaAssets.map((media) => [media.id, media]),
);
const pendingElements: Array<Promise<CollectedAudioElement | null>> = [];
for (const track of orderedTracks) { for (const track of allTracks) {
if (canTrackHaveAudio(track) && track.muted) continue; if (canTrackHaveAudio(track) && track.muted) continue;
for (const element of track.elements) { for (const element of track.elements) {
@ -110,65 +110,95 @@ export async function collectAudioElements({
: null; : null;
if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue;
const isTrackMuted = canTrackHaveAudio(track) && track.muted; candidates.push({ element, mediaAsset });
}
}
if (element.type === "audio") { return candidates;
pendingElements.push( }
resolveAudioBufferForElement({
element,
mediaMap,
audioContext,
}).then((audioBuffer) => {
if (!audioBuffer) return null;
const muted = element.muted === true || isTrackMuted;
return {
timelineElement: element,
buffer: audioBuffer,
startTime: element.startTime / TICKS_PER_SECOND,
duration: element.duration / TICKS_PER_SECOND,
trimStart: element.trimStart / TICKS_PER_SECOND,
trimEnd: element.trimEnd / TICKS_PER_SECOND,
volume: resolveEffectiveAudioGain({
element,
trackMuted: isTrackMuted,
localTime: 0,
}),
muted,
retime: element.retime,
};
}),
);
continue;
}
if (element.type === "video") { export function timelineHasAudio({
if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue; tracks,
mediaAssets,
}: {
tracks: SceneTracks;
mediaAssets: MediaAsset[];
}): boolean {
return collectAudibleCandidates({ tracks, mediaAssets }).some(
({ element }) => element.muted !== true,
);
}
pendingElements.push( export async function collectAudioElements({
resolveAudioBufferForVideoElement({ tracks,
mediaAsset, mediaAssets,
audioContext, audioContext,
}).then((audioBuffer) => { }: {
if (!audioBuffer) return null; tracks: SceneTracks;
const muted = (element.muted ?? false) || isTrackMuted; mediaAssets: MediaAsset[];
return { audioContext: AudioContext;
timelineElement: element, }): Promise<CollectedAudioElement[]> {
buffer: audioBuffer, const candidates = collectAudibleCandidates({ tracks, mediaAssets });
startTime: element.startTime / TICKS_PER_SECOND, const mediaMap = new Map<string, MediaAsset>(
duration: element.duration / TICKS_PER_SECOND, mediaAssets.map((media) => [media.id, media]),
trimStart: element.trimStart / TICKS_PER_SECOND, );
trimEnd: element.trimEnd / TICKS_PER_SECOND, const pendingElements: Array<Promise<CollectedAudioElement | null>> = [];
volume: resolveEffectiveAudioGain({
element, for (const { element, mediaAsset } of candidates) {
trackMuted: isTrackMuted, if (element.type === "audio") {
localTime: 0, pendingElements.push(
}), resolveAudioBufferForElement({
muted, element,
retime: element.retime, mediaMap,
}; audioContext,
}), }).then((audioBuffer) => {
); if (!audioBuffer) return null;
} return {
timelineElement: element,
buffer: audioBuffer,
startTime: element.startTime / TICKS_PER_SECOND,
duration: element.duration / TICKS_PER_SECOND,
trimStart: element.trimStart / TICKS_PER_SECOND,
trimEnd: element.trimEnd / TICKS_PER_SECOND,
volume: resolveEffectiveAudioGain({
element,
trackMuted: false,
localTime: 0,
}),
muted: element.muted === true,
retime: element.retime,
};
}),
);
continue;
}
if (element.type === "video") {
if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue;
pendingElements.push(
resolveAudioBufferForVideoElement({
mediaAsset,
audioContext,
}).then((audioBuffer) => {
if (!audioBuffer) return null;
return {
timelineElement: element,
buffer: audioBuffer,
startTime: element.startTime / TICKS_PER_SECOND,
duration: element.duration / TICKS_PER_SECOND,
trimStart: element.trimStart / TICKS_PER_SECOND,
trimEnd: element.trimEnd / TICKS_PER_SECOND,
volume: resolveEffectiveAudioGain({
element,
trackMuted: false,
localTime: 0,
}),
muted: element.muted ?? false,
retime: element.retime,
};
}),
);
} }
} }
@ -339,16 +369,16 @@ async function fetchLibraryAudioSource({
type: "audio/mpeg", type: "audio/mpeg",
}); });
return { return {
timelineElement: element, timelineElement: element,
file, file,
startTime: element.startTime / TICKS_PER_SECOND, startTime: element.startTime / TICKS_PER_SECOND,
duration: element.duration / TICKS_PER_SECOND, duration: element.duration / TICKS_PER_SECOND,
trimStart: element.trimStart / TICKS_PER_SECOND, trimStart: element.trimStart / TICKS_PER_SECOND,
trimEnd: element.trimEnd / TICKS_PER_SECOND, trimEnd: element.trimEnd / TICKS_PER_SECOND,
volume, volume,
retime: element.retime, retime: element.retime,
}; };
} catch (error) { } catch (error) {
console.warn("Failed to fetch library audio:", error); console.warn("Failed to fetch library audio:", error);
return null; return null;

326
apps/web/src/lib/media/mediabunny.ts
View File

@ -1,158 +1,168 @@
import { Input, ALL_FORMATS, BlobSource } from "mediabunny"; import { Input, ALL_FORMATS, BlobSource } from "mediabunny";
import { createTimelineAudioBuffer } from "@/lib/media/audio"; import { createTimelineAudioBuffer } from "@/lib/media/audio";
import type { SceneTracks } from "@/lib/timeline"; import type { SceneTracks } from "@/lib/timeline";
import type { MediaAsset } from "@/lib/media/types"; import type { MediaAsset } from "@/lib/media/types";
import { TICKS_PER_SECOND } from "@/lib/wasm";
export async function getVideoInfo({
videoFile, export async function getVideoInfo({
}: { videoFile,
videoFile: File; }: {
}): Promise<{ videoFile: File;
duration: number; }): Promise<{
width: number; duration: number;
height: number; width: number;
fps: number; height: number;
hasAudio: boolean; fps: number;
}> { hasAudio: boolean;
const input = new Input({ }> {
source: new BlobSource(videoFile), const input = new Input({
formats: ALL_FORMATS, source: new BlobSource(videoFile),
}); formats: ALL_FORMATS,
});
const duration = await input.computeDuration();
const videoTrack = await input.getPrimaryVideoTrack(); const duration = await input.computeDuration();
const videoTrack = await input.getPrimaryVideoTrack();
if (!videoTrack) {
throw new Error("No video track found in the file"); if (!videoTrack) {
} throw new Error("No video track found in the file");
}
const packetStats = await videoTrack.computePacketStats(100);
const fps = packetStats.averagePacketRate; const packetStats = await videoTrack.computePacketStats(100);
const audioTrack = await input.getPrimaryAudioTrack(); const fps = packetStats.averagePacketRate;
const audioTrack = await input.getPrimaryAudioTrack();
return {
duration, return {
width: videoTrack.displayWidth, duration,
height: videoTrack.displayHeight, width: videoTrack.displayWidth,
fps, height: videoTrack.displayHeight,
hasAudio: audioTrack !== null, fps,
}; hasAudio: audioTrack !== null,
} };
}
const SAMPLE_RATE = 44100;
const NUM_CHANNELS = 2; const SAMPLE_RATE = 44100;
const NUM_CHANNELS = 2;
export const extractTimelineAudio = async ({ const EMPTY_TIMELINE_SILENT_DURATION_SECONDS = 0.1;
tracks, const MIN_SILENT_DURATION_SECONDS = 0.001;
mediaAssets,
totalDuration, export const extractTimelineAudio = async ({
onProgress, tracks,
}: { mediaAssets,
tracks: SceneTracks; totalDuration,
mediaAssets: MediaAsset[]; onProgress,
totalDuration: number; }: {
onProgress?: (progress: number) => void; tracks: SceneTracks;
}): Promise<Blob> => { mediaAssets: MediaAsset[];
if (totalDuration === 0) { totalDuration: number;
return createWavBlob({ samples: new Float32Array(SAMPLE_RATE * 0.1) }); onProgress?: (progress: number) => void;
} }): Promise<Blob> => {
if (totalDuration === 0) {
onProgress?.(10); return createWavBlob({
samples: new Float32Array(
const audioBuffer = await createTimelineAudioBuffer({ SAMPLE_RATE * EMPTY_TIMELINE_SILENT_DURATION_SECONDS,
tracks, ),
mediaAssets, });
duration: totalDuration, }
sampleRate: SAMPLE_RATE,
}); onProgress?.(10);
if (!audioBuffer) { const audioBuffer = await createTimelineAudioBuffer({
const silentDuration = Math.max(1, totalDuration); tracks,
const silentSamples = new Float32Array( mediaAssets,
Math.ceil(silentDuration * SAMPLE_RATE) * NUM_CHANNELS, duration: totalDuration,
); sampleRate: SAMPLE_RATE,
return createWavBlob({ samples: silentSamples }); });
}
if (!audioBuffer) {
onProgress?.(90); const silentDurationSeconds = Math.max(
MIN_SILENT_DURATION_SECONDS,
const interleavedSamples = interleaveAudioBuffer({ audioBuffer }); totalDuration / TICKS_PER_SECOND,
onProgress?.(100); );
const silentSamples = new Float32Array(
return createWavBlob({ samples: interleavedSamples }); Math.ceil(silentDurationSeconds * SAMPLE_RATE) * NUM_CHANNELS,
}; );
return createWavBlob({ samples: silentSamples });
function interleaveAudioBuffer({ }
audioBuffer,
}: { onProgress?.(90);
audioBuffer: AudioBuffer;
}): Float32Array { const interleavedSamples = interleaveAudioBuffer({ audioBuffer });
const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels); onProgress?.(100);
const interleavedSamples = new Float32Array(
audioBuffer.length * NUM_CHANNELS, return createWavBlob({ samples: interleavedSamples });
); };
for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) { function interleaveAudioBuffer({
for (let channel = 0; channel < NUM_CHANNELS; channel++) { audioBuffer,
const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1)); }: {
interleavedSamples[sampleIndex * NUM_CHANNELS + channel] = audioBuffer: AudioBuffer;
audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0; }): Float32Array {
} const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels);
} const interleavedSamples = new Float32Array(
audioBuffer.length * NUM_CHANNELS,
return interleavedSamples; );
}
for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {
function createWavBlob({ samples }: { samples: Float32Array }): Blob { for (let channel = 0; channel < NUM_CHANNELS; channel++) {
const numChannels = NUM_CHANNELS; const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1));
const bitsPerSample = 16; interleavedSamples[sampleIndex * NUM_CHANNELS + channel] =
const bytesPerSample = bitsPerSample / 8; audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0;
const numSamples = samples.length / numChannels; }
const dataSize = numSamples * numChannels * bytesPerSample; }
const buffer = new ArrayBuffer(44 + dataSize);
const view = new DataView(buffer); return interleavedSamples;
}
// riff header
writeString({ view, offset: 0, str: "RIFF" }); function createWavBlob({ samples }: { samples: Float32Array }): Blob {
view.setUint32(4, 36 + dataSize, true); const numChannels = NUM_CHANNELS;
writeString({ view, offset: 8, str: "WAVE" }); const bitsPerSample = 16;
const bytesPerSample = bitsPerSample / 8;
// fmt chunk const numSamples = samples.length / numChannels;
writeString({ view, offset: 12, str: "fmt " }); const dataSize = numSamples * numChannels * bytesPerSample;
view.setUint32(16, 16, true); const buffer = new ArrayBuffer(44 + dataSize);
view.setUint16(20, 1, true); const view = new DataView(buffer);
view.setUint16(22, numChannels, true);
view.setUint32(24, SAMPLE_RATE, true); // riff header
view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true); writeString({ view, offset: 0, str: "RIFF" });
view.setUint16(32, numChannels * bytesPerSample, true); view.setUint32(4, 36 + dataSize, true);
view.setUint16(34, bitsPerSample, true); writeString({ view, offset: 8, str: "WAVE" });
// data chunk // fmt chunk
writeString({ view, offset: 36, str: "data" }); writeString({ view, offset: 12, str: "fmt " });
view.setUint32(40, dataSize, true); view.setUint32(16, 16, true);
view.setUint16(20, 1, true);
// convert float32 to int16 and write view.setUint16(22, numChannels, true);
let offset = 44; view.setUint32(24, SAMPLE_RATE, true);
for (let i = 0; i < samples.length; i++) { view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true);
const sample = Math.max(-1, Math.min(1, samples[i])); view.setUint16(32, numChannels * bytesPerSample, true);
const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff; view.setUint16(34, bitsPerSample, true);
view.setInt16(offset, int16, true);
offset += 2; // data chunk
} writeString({ view, offset: 36, str: "data" });
view.setUint32(40, dataSize, true);
return new Blob([buffer], { type: "audio/wav" });
} // convert float32 to int16 and write
let offset = 44;
function writeString({ for (let i = 0; i < samples.length; i++) {
view, const sample = Math.max(-1, Math.min(1, samples[i]));
offset, const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
str, view.setInt16(offset, int16, true);
}: { offset += 2;
view: DataView; }
offset: number;
str: string; return new Blob([buffer], { type: "audio/wav" });
}): void { }
for (let i = 0; i < str.length; i++) {
view.setUint8(offset + i, str.charCodeAt(i)); function writeString({
} view,
} offset,
str,
}: {
view: DataView;
offset: number;
str: string;
}): void {
for (let i = 0; i < str.length; i++) {
view.setUint8(offset + i, str.charCodeAt(i));
}
}

14
apps/web/src/lib/timeline/audio-separation/index.ts
View File

@ -49,7 +49,9 @@ export function canToggleSourceAudio(
element: TimelineElement, element: TimelineElement,
mediaAsset: MediaAudioState | null | undefined, mediaAsset: MediaAudioState | null | undefined,
): element is VideoElement { ): element is VideoElement {
return canRecoverSourceAudio(element) || canExtractSourceAudio(element, mediaAsset); return (
canRecoverSourceAudio(element) || canExtractSourceAudio(element, mediaAsset)
);
} }
export function doesElementHaveEnabledAudio({ export function doesElementHaveEnabledAudio({
@ -63,7 +65,11 @@ export function doesElementHaveEnabledAudio({
return true; return true;
} }
return !!mediaAsset && mediaAsset.hasAudio !== false && isSourceAudioEnabled({ element }); return (
!!mediaAsset &&
mediaAsset.hasAudio !== false &&
isSourceAudioEnabled({ element })
);
} }
export function buildSeparatedAudioElement({ export function buildSeparatedAudioElement({
@ -100,7 +106,9 @@ export function getSourceAudioActionLabel({
}: { }: {
element: VideoElement; element: VideoElement;
}): "Extract audio" | "Recover audio" { }): "Extract audio" | "Recover audio" {
return isSourceAudioSeparated({ element }) ? "Recover audio" : "Extract audio"; return isSourceAudioSeparated({ element })
? "Recover audio"
: "Extract audio";
} }
function cloneVolumeAnimations({ function cloneVolumeAnimations({

25
apps/web/src/lib/transcription/diagnostics.ts Normal file
View File

@ -0,0 +1,25 @@
import type { DiagnosticsManager } from "@/core/managers/diagnostics-manager";
import { timelineHasAudio } from "@/lib/media/audio";
export const TRANSCRIPTION_DIAGNOSTICS_SCOPE = "transcription";
export function registerTranscriptionDiagnostics({
diagnostics,
}: {
diagnostics: DiagnosticsManager;
}): void {
diagnostics.register({
id: "transcription.no_audio",
scope: TRANSCRIPTION_DIAGNOSTICS_SCOPE,
severity: "caution",
message: "No audio detected. Add a clip with audio to the timeline first.",
check: (editor) => {
const scene = editor.scenes.getActiveSceneOrNull();
if (!scene) return false;
return !timelineHasAudio({
tracks: scene.tracks,
mediaAssets: editor.media.getAssets(),
});
},
});
}