From 91e1232112421438ce73b9361896f9b58c84fb3e Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Tue, 14 Apr 2026 23:34:57 +0200 Subject: [PATCH] feat: transcription diagnostics --- apps/web/src/app/globals.css | 575 +++++------ .../editor/panels/assets/views/captions.tsx | 99 +- .../editor/panels/assets/views/stickers.tsx | 910 +++++++++--------- apps/web/src/components/ui/button.tsx | 1 + apps/web/src/core/index.ts | 5 + .../src/core/managers/diagnostics-manager.ts | 38 + apps/web/src/hooks/use-editor.ts | 1 + apps/web/src/lib/diagnostics/types.ts | 8 + apps/web/src/lib/media/audio.ts | 188 ++-- apps/web/src/lib/media/mediabunny.ts | 326 ++++--- .../lib/timeline/audio-separation/index.ts | 14 +- apps/web/src/lib/transcription/diagnostics.ts | 25 + 12 files changed, 1190 insertions(+), 1000 deletions(-) create mode 100644 apps/web/src/core/managers/diagnostics-manager.ts create mode 100644 apps/web/src/lib/diagnostics/types.ts create mode 100644 apps/web/src/lib/transcription/diagnostics.ts diff --git a/apps/web/src/app/globals.css b/apps/web/src/app/globals.css index a2df844a..36417e15 100644 --- a/apps/web/src/app/globals.css +++ b/apps/web/src/app/globals.css @@ -1,284 +1,291 @@ -@import "tailwindcss"; - -/* Custom variant for dark mode */ -@custom-variant dark (&:where(.dark, .dark *)); - -/* Plugins */ -@plugin "@tailwindcss/typography"; -@plugin "tailwindcss-animate"; - -:root { - --background: hsl(0, 0%, 100%); - --foreground: hsl(0 0% 11%); - --card: hsl(0, 0%, 100%); - --card-foreground: hsl(0 0% 11%); - --popover: hsl(0, 0%, 100%); - --popover-hover: hsl(0, 0%, 96%); - --popover-foreground: hsl(0 0% 2%); - --primary: hsl(200, 90%, 52%); - --primary-foreground: hsl(0, 0%, 100%); - --secondary: hsl(204, 100%, 97%); - --secondary-border: hsl(204, 100%, 94%); - --secondary-foreground: hsl(200, 98%, 39%); - --muted: hsl(0 0% 85.1%); - --muted-foreground: hsl(0 0% 50%); - --accent: hsl(0, 0%, 96%); - --accent-foreground: hsl(0 0% 2%); - --destructive: hsl(0, 83%, 50%); - --destructive-foreground: hsl(0, 0%, 100%); - --constructive: hsl(141, 71%, 48%); - --constructive-foreground: hsl(0, 0%, 100%); - --border: hsl(0 0% 91%); - --input: hsl(0, 0%, 100%); - --ring: hsl(0, 0%, 55%); - --chart-1: hsl(220 70% 50%); - --chart-2: hsl(160 60% 45%); - --chart-3: hsl(30 80% 55%); - --chart-4: hsl(280 65% 60%); - --chart-5: hsl(340 75% 55%); - --sidebar-background: hsl(0 0% 96.1%); - --sidebar-foreground: hsl(0 0% 2%); - --sidebar-primary: hsl(0 0% 2%); - --sidebar-primary-foreground: hsl(0 0% 91%); - --sidebar-accent: hsl(0 0% 85.1%); - --sidebar-accent-foreground: hsl(0 0% 2%); - --sidebar-border: hsl(0 0% 85.1%); - --sidebar-ring: hsl(0 0% 16.9%); - --sidebar: hsl(0 0% 98%); -} - -.panel { - --background: hsl(210, 20%, 98%); - --foreground: hsl(0 0% 13%); - --card: hsl(0, 0%, 98%); - --card-foreground: hsl(0 0% 13%); - --primary-foreground: hsl(0, 0%, 98%); - --secondary: hsl(204, 100%, 95%); - --secondary-border: hsl(204, 100%, 92%); - --secondary-foreground: hsl(200, 98%, 37%); - --muted: hsl(0 0% 83.1%); - --muted-foreground: hsl(0 0% 48%); - --accent: hsl(0, 0%, 93%); - --accent-foreground: hsl(0 0% 5%); - --destructive-foreground: hsl(0, 0%, 98%); - --constructive-foreground: hsl(0, 0%, 98%); - --border: hsl(0 0% 87%); - --input: hsl(0 0% 93%); - --ring: hsl(0, 0%, 53%); -} - -.dark { - --background: hsl(0, 0%, 5%); - --foreground: hsl(0 0% 87%); - --card: hsl(0, 0%, 5%); - --card-foreground: hsl(0 0% 87%); - --popover: hsl(0, 0%, 5%); - --popover-hover: hsl(0, 0%, 13%); - --popover-foreground: hsl(0 0% 95%); - --secondary: hsl(204, 100%, 12%); - --secondary-border: hsl(204, 100%, 15%); - --secondary-foreground: hsl(200, 98%, 61%); - --muted: hsl(0 0% 20%); - --accent: hsl(0, 0%, 14%); - --accent-foreground: hsl(0 0% 95%); - --border: hsl(0 0% 16%); - --input: hsl(0 0% 5%); - --ring: hsl(0, 0%, 50%); - --sidebar-background: hsl(0 0% 8%); - --sidebar-foreground: hsl(0 0% 95%); - --sidebar-primary: hsl(0 0% 95%); - --sidebar-primary-foreground: hsl(0 0% 15%); - --sidebar-accent: hsl(0 0% 20%); - --sidebar-accent-foreground: hsl(0 0% 95%); - --sidebar-border: hsl(0 0% 20%); - --sidebar-ring: hsl(0 0% 83.1%); - --sidebar: hsl(0 0% 6%); -} - -.dark .panel { - --background: hsl(0 0% 10%); - --foreground: hsl(0 0% 85%); - --card: hsl(0, 0%, 10%); - --card-foreground: hsl(0 0% 85%); - --secondary: hsl(204, 67%, 9%); - --secondary-border: hsl(204, 100%, 14%); - --secondary-foreground: hsl(200, 98%, 63%); - --muted: hsl(0 0% 22%); - --accent: hsl(0, 0%, 15%); - --accent-foreground: hsl(0 0% 93%); - --border: hsl(0 0% 18%); - --input: hsl(0 0% 22%); - --ring: hsl(0, 0%, 52%); -} - -@layer base { - /* - The default border color has changed to `currentcolor` in Tailwind CSS v4, - so we've added these compatibility styles to make sure everything still - looks the same as it did with Tailwind CSS v3. - - If we ever want to remove these styles, we need to add an explicit border - color utility to any element that depends on these defaults. - */ - *, - ::after, - ::before, - ::backdrop, - ::file-selector-button { - border-color: var(--color-gray-200, currentcolor); - } - /* Other default base styles */ - * { - @apply border-border; - } - body { - @apply bg-background text-foreground; - /* Prevent back/forward swipe */ - overscroll-behavior-x: contain; - } - ::selection { - @apply bg-primary/35 selection:text-primary-foreground; - } -} - -@theme inline { - /* Responsive breakpoints */ - --breakpoint-xs: 30rem; - - /* Typography */ - --font-sans: var(--font-inter), sans-serif; - - /* Font sizes */ - --text-xs: 0.72rem; - --text-sm: 0.79rem; - --text-base: 0.92rem; - --text-base--line-height: calc(1.5 / 0.95); - --text-xs--line-height: calc(1 / 0.8); - - /* Border radius */ - --radius-lg: 0.82rem; - --radius-md: 0.65rem; - --radius-sm: 0.35rem; - - /* Palette mapped to root design tokens */ - --color-background: var(--background); - --color-foreground: var(--foreground); - - --color-card: var(--card); - --color-card-foreground: var(--card-foreground); - - --color-popover: var(--popover); - --color-popover-hover: var(--popover-hover); - --color-popover-foreground: var(--popover-foreground); - - --color-primary: var(--primary); - --color-primary-foreground: var(--primary-foreground); - --color-secondary: var(--secondary); - --color-secondary-border: var(--secondary-border); - --color-secondary-foreground: var(--secondary-foreground); - - --color-muted: var(--muted); - --color-muted-foreground: var(--muted-foreground); - - --color-accent: var(--accent); - --color-accent-foreground: var(--accent-foreground); - - --color-destructive: var(--destructive); - --color-destructive-foreground: var(--destructive-foreground); - - --color-constructive: var(--constructive); - --color-constructive-foreground: var(--constructive-foreground); - - --color-border: var(--border); - --color-input: var(--input); - --color-ring: var(--ring); - - /* Chart colors */ - --color-chart-1: var(--chart-1); - --color-chart-2: var(--chart-2); - --color-chart-3: var(--chart-3); - --color-chart-4: var(--chart-4); - --color-chart-5: var(--chart-5); - - /* Sidebar */ - --color-sidebar: var(--sidebar-background); - --color-sidebar-foreground: var(--sidebar-foreground); - --color-sidebar-primary: var(--sidebar-primary); - --color-sidebar-primary-foreground: var(--sidebar-primary-foreground); - --color-sidebar-accent: var(--sidebar-accent); - --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); - --color-sidebar-border: var(--sidebar-border); - --color-sidebar-ring: var(--sidebar-ring); - - /* Animations */ - --animate-accordion-down: accordion-down 0.2s ease-out; - --animate-accordion-up: accordion-up 0.2s ease-out; - - @keyframes accordion-down { - from { - height: 0; - } - to { - height: var(--radix-accordion-content-height); - } - } - - @keyframes accordion-up { - from { - height: var(--radix-accordion-content-height); - } - to { - height: 0; - } - } -} - -@utility scrollbar-hidden { - -ms-overflow-style: none; - scrollbar-width: none; - &::-webkit-scrollbar { - display: none; - } -} - -@utility scrollbar-x-hidden { - -ms-overflow-style: none; - scrollbar-width: none; - &::-webkit-scrollbar:horizontal { - display: none; - } -} - -@utility scrollbar-y-hidden { - -ms-overflow-style: none; - scrollbar-width: none; - &::-webkit-scrollbar:vertical { - display: none; - } -} - -@utility scrollbar-thin { - &::-webkit-scrollbar { - width: 7px; - height: 7px; - } - &::-webkit-scrollbar-track { - background: transparent; - } - &::-webkit-scrollbar-thumb { - background: var(--border); - border-radius: 4px; - } - &::-webkit-scrollbar-thumb:hover { - background: var(--muted-foreground); - } -} - -@layer base { - * { - @apply border-border outline-ring/50; - } - body { - @apply bg-background text-foreground; - } -} +@import "tailwindcss"; + +/* Custom variant for dark mode */ +@custom-variant dark (&:where(.dark, .dark *)); + +/* Plugins */ +@plugin "@tailwindcss/typography"; +@plugin "tailwindcss-animate"; + +:root { + --background: hsl(0, 0%, 100%); + --foreground: hsl(0 0% 11%); + --card: hsl(0, 0%, 100%); + --card-foreground: hsl(0 0% 11%); + --popover: hsl(0, 0%, 100%); + --popover-hover: hsl(0, 0%, 96%); + --popover-foreground: hsl(0 0% 2%); + --primary: hsl(200, 90%, 52%); + --primary-foreground: hsl(0, 0%, 100%); + --secondary: hsl(204, 100%, 97%); + --secondary-border: hsl(204, 100%, 94%); + --secondary-foreground: hsl(200, 98%, 39%); + --muted: hsl(0 0% 85.1%); + --muted-foreground: hsl(0 0% 50%); + --accent: hsl(0, 0%, 96%); + --accent-foreground: hsl(0 0% 2%); + --destructive: hsl(0, 83%, 50%); + --destructive-foreground: hsl(0, 0%, 100%); + --constructive: hsl(141, 71%, 48%); + --constructive-foreground: hsl(0, 0%, 100%); + --caution: hsl(38, 92%, 50%); + --caution-foreground: hsl(0, 0%, 10%); + --border: hsl(0 0% 91%); + --input: hsl(0, 0%, 100%); + --ring: hsl(0, 0%, 55%); + --chart-1: hsl(220 70% 50%); + --chart-2: hsl(160 60% 45%); + --chart-3: hsl(30 80% 55%); + --chart-4: hsl(280 65% 60%); + --chart-5: hsl(340 75% 55%); + --sidebar-background: hsl(0 0% 96.1%); + --sidebar-foreground: hsl(0 0% 2%); + --sidebar-primary: hsl(0 0% 2%); + --sidebar-primary-foreground: hsl(0 0% 91%); + --sidebar-accent: hsl(0 0% 85.1%); + --sidebar-accent-foreground: hsl(0 0% 2%); + --sidebar-border: hsl(0 0% 85.1%); + --sidebar-ring: hsl(0 0% 16.9%); + --sidebar: hsl(0 0% 98%); +} + +.panel { + --background: hsl(210, 20%, 98%); + --foreground: hsl(0 0% 13%); + --card: hsl(0, 0%, 98%); + --card-foreground: hsl(0 0% 13%); + --primary-foreground: hsl(0, 0%, 98%); + --secondary: hsl(204, 100%, 95%); + --secondary-border: hsl(204, 100%, 92%); + --secondary-foreground: hsl(200, 98%, 37%); + --muted: hsl(0 0% 83.1%); + --muted-foreground: hsl(0 0% 48%); + --accent: hsl(0, 0%, 93%); + --accent-foreground: hsl(0 0% 5%); + --destructive-foreground: hsl(0, 0%, 98%); + --constructive-foreground: hsl(0, 0%, 98%); + --border: hsl(0 0% 87%); + --input: hsl(0 0% 93%); + --ring: hsl(0, 0%, 53%); +} + +.dark { + --background: hsl(0, 0%, 5%); + --foreground: hsl(0 0% 87%); + --card: hsl(0, 0%, 5%); + --card-foreground: hsl(0 0% 87%); + --popover: hsl(0, 0%, 5%); + --popover-hover: hsl(0, 0%, 13%); + --popover-foreground: hsl(0 0% 95%); + --secondary: hsl(204, 100%, 12%); + --secondary-border: hsl(204, 100%, 15%); + --secondary-foreground: hsl(200, 98%, 61%); + --muted: hsl(0 0% 20%); + --accent: hsl(0, 0%, 14%); + --accent-foreground: hsl(0 0% 95%); + --border: hsl(0 0% 16%); + --input: hsl(0 0% 5%); + --ring: hsl(0, 0%, 50%); + --caution: hsl(38, 92%, 60%); + --caution-foreground: hsl(0, 0%, 10%); + --sidebar-background: hsl(0 0% 8%); + --sidebar-foreground: hsl(0 0% 95%); + --sidebar-primary: hsl(0 0% 95%); + --sidebar-primary-foreground: hsl(0 0% 15%); + --sidebar-accent: hsl(0 0% 20%); + --sidebar-accent-foreground: hsl(0 0% 95%); + --sidebar-border: hsl(0 0% 20%); + --sidebar-ring: hsl(0 0% 83.1%); + --sidebar: hsl(0 0% 6%); +} + +.dark .panel { + --background: hsl(0 0% 10%); + --foreground: hsl(0 0% 85%); + --card: hsl(0, 0%, 10%); + --card-foreground: hsl(0 0% 85%); + --secondary: hsl(204, 67%, 9%); + --secondary-border: hsl(204, 100%, 14%); + --secondary-foreground: hsl(200, 98%, 63%); + --muted: hsl(0 0% 22%); + --accent: hsl(0, 0%, 15%); + --accent-foreground: hsl(0 0% 93%); + --border: hsl(0 0% 18%); + --input: hsl(0 0% 22%); + --ring: hsl(0, 0%, 52%); +} + +@layer base { + /* + The default border color has changed to `currentcolor` in Tailwind CSS v4, + so we've added these compatibility styles to make sure everything still + looks the same as it did with Tailwind CSS v3. + + If we ever want to remove these styles, we need to add an explicit border + color utility to any element that depends on these defaults. + */ + *, + ::after, + ::before, + ::backdrop, + ::file-selector-button { + border-color: var(--color-gray-200, currentcolor); + } + /* Other default base styles */ + * { + @apply border-border; + } + body { + @apply bg-background text-foreground; + /* Prevent back/forward swipe */ + overscroll-behavior-x: contain; + } + ::selection { + @apply bg-primary/35 selection:text-primary-foreground; + } +} + +@theme inline { + /* Responsive breakpoints */ + --breakpoint-xs: 30rem; + + /* Typography */ + --font-sans: var(--font-inter), sans-serif; + + /* Font sizes */ + --text-xs: 0.72rem; + --text-sm: 0.79rem; + --text-base: 0.92rem; + --text-base--line-height: calc(1.5 / 0.95); + --text-xs--line-height: calc(1 / 0.8); + + /* Border radius */ + --radius-lg: 0.82rem; + --radius-md: 0.65rem; + --radius-sm: 0.35rem; + + /* Palette mapped to root design tokens */ + --color-background: var(--background); + --color-foreground: var(--foreground); + + --color-card: var(--card); + --color-card-foreground: var(--card-foreground); + + --color-popover: var(--popover); + --color-popover-hover: var(--popover-hover); + --color-popover-foreground: var(--popover-foreground); + + --color-primary: var(--primary); + --color-primary-foreground: var(--primary-foreground); + --color-secondary: var(--secondary); + --color-secondary-border: var(--secondary-border); + --color-secondary-foreground: var(--secondary-foreground); + + --color-muted: var(--muted); + --color-muted-foreground: var(--muted-foreground); + + --color-accent: var(--accent); + --color-accent-foreground: var(--accent-foreground); + + --color-destructive: var(--destructive); + --color-destructive-foreground: var(--destructive-foreground); + + --color-constructive: var(--constructive); + --color-constructive-foreground: var(--constructive-foreground); + + --color-caution: var(--caution); + --color-caution-foreground: var(--caution-foreground); + + --color-border: var(--border); + --color-input: var(--input); + --color-ring: var(--ring); + + /* Chart colors */ + --color-chart-1: var(--chart-1); + --color-chart-2: var(--chart-2); + --color-chart-3: var(--chart-3); + --color-chart-4: var(--chart-4); + --color-chart-5: var(--chart-5); + + /* Sidebar */ + --color-sidebar: var(--sidebar-background); + --color-sidebar-foreground: var(--sidebar-foreground); + --color-sidebar-primary: var(--sidebar-primary); + --color-sidebar-primary-foreground: var(--sidebar-primary-foreground); + --color-sidebar-accent: var(--sidebar-accent); + --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); + --color-sidebar-border: var(--sidebar-border); + --color-sidebar-ring: var(--sidebar-ring); + + /* Animations */ + --animate-accordion-down: accordion-down 0.2s ease-out; + --animate-accordion-up: accordion-up 0.2s ease-out; + + @keyframes accordion-down { + from { + height: 0; + } + to { + height: var(--radix-accordion-content-height); + } + } + + @keyframes accordion-up { + from { + height: var(--radix-accordion-content-height); + } + to { + height: 0; + } + } +} + +@utility scrollbar-hidden { + -ms-overflow-style: none; + scrollbar-width: none; + &::-webkit-scrollbar { + display: none; + } +} + +@utility scrollbar-x-hidden { + -ms-overflow-style: none; + scrollbar-width: none; + &::-webkit-scrollbar:horizontal { + display: none; + } +} + +@utility scrollbar-y-hidden { + -ms-overflow-style: none; + scrollbar-width: none; + &::-webkit-scrollbar:vertical { + display: none; + } +} + +@utility scrollbar-thin { + &::-webkit-scrollbar { + width: 7px; + height: 7px; + } + &::-webkit-scrollbar-track { + background: transparent; + } + &::-webkit-scrollbar-thumb { + background: var(--border); + border-radius: 4px; + } + &::-webkit-scrollbar-thumb:hover { + background: var(--muted-foreground); + } +} + +@layer base { + * { + @apply border-border outline-ring/50; + } + body { + @apply bg-background text-foreground; + } +} diff --git a/apps/web/src/components/editor/panels/assets/views/captions.tsx b/apps/web/src/components/editor/panels/assets/views/captions.tsx index 7b6ae5ca..9e394cfc 100644 --- a/apps/web/src/components/editor/panels/assets/views/captions.tsx +++ b/apps/web/src/components/editor/panels/assets/views/captions.tsx @@ -10,9 +10,8 @@ import { import { useReducer, useRef, useState } from "react"; import { extractTimelineAudio } from "@/lib/media/mediabunny"; import { useEditor } from "@/hooks/use-editor"; -import { - DEFAULT_TRANSCRIPTION_SAMPLE_RATE, -} from "@/lib/transcription/audio"; +import { TRANSCRIPTION_DIAGNOSTICS_SCOPE } from "@/lib/transcription/diagnostics"; +import { DEFAULT_TRANSCRIPTION_SAMPLE_RATE } from "@/lib/transcription/audio"; import { TRANSCRIPTION_LANGUAGES } from "@/lib/transcription/supported-languages"; import type { CaptionChunk, @@ -31,8 +30,23 @@ import { SectionField, SectionFields, } from "@/components/section"; -import { CloudUploadIcon } from "@hugeicons/core-free-icons"; +import { AlertCircleIcon, CloudUploadIcon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import type { DiagnosticSeverity } from "@/lib/diagnostics/types"; + +const DIAGNOSTIC_BUTTON_VARIANT: Record< + DiagnosticSeverity, + "caution" | "destructive-foreground" +> = { + caution: "caution", + error: "destructive-foreground", +}; type ProcessingState = | { status: "idle"; error: string | null; warnings: string[] } @@ -44,7 +58,11 @@ type ProcessingAction = | { type: "succeed"; warnings: string[] } | { type: "fail"; error: string }; -const IDLE_STATE: ProcessingState = { status: "idle", error: null, warnings: [] }; +const IDLE_STATE: ProcessingState = { + status: "idle", + error: null, + warnings: [], +}; function processingReducer( state: ProcessingState, @@ -73,6 +91,10 @@ export function Captions() { const isProcessing = processing.status === "processing"; + const activeDiagnostics = useEditor((e) => + e.diagnostics.getActive({ scope: TRANSCRIPTION_DIAGNOSTICS_SCOPE }), + ); + const handleProgress = (progress: TranscriptionProgress) => { if (progress.status === "loading-model") { dispatch({ @@ -84,7 +106,11 @@ export function Captions() { } }; - const insertCaptions = ({ captions }: { captions: CaptionChunk[] }): boolean => { + const insertCaptions = ({ + captions, + }: { + captions: CaptionChunk[]; + }): boolean => { const trackId = insertCaptionChunksAsTextTrack({ editor, captions }); return trackId !== null; }; @@ -123,7 +149,10 @@ export function Captions() { console.error("Transcription failed:", error); dispatch({ type: "fail", - error: error instanceof Error ? error.message : "An unexpected error occurred", + error: + error instanceof Error + ? error.message + : "An unexpected error occurred", }); } }; @@ -168,7 +197,10 @@ export function Captions() { console.error("Subtitle import failed:", error); dispatch({ type: "fail", - error: error instanceof Error ? error.message : "An unexpected error occurred", + error: + error instanceof Error + ? error.message + : "An unexpected error occurred", }); } }; @@ -208,17 +240,38 @@ export function Captions() { title="Captions" contentClassName="px-0 flex flex-col h-full" actions={ - + +
+ {!isProcessing && + activeDiagnostics.map((diagnostic) => ( + + + + + + {diagnostic.message} + + + ))} + +
+ } ref={containerRef} > @@ -229,7 +282,11 @@ export function Captions() { className="hidden" onChange={(event) => void handleFileChange({ event })} /> -
+
@@ -256,7 +313,7 @@ export function Captions() { type="button" className="mt-auto w-full" onClick={handleGenerateTranscript} - disabled={isProcessing} + disabled={isProcessing || activeDiagnostics.length > 0} > {isProcessing && } {isProcessing ? processing.step : "Generate transcript"} diff --git a/apps/web/src/components/editor/panels/assets/views/stickers.tsx b/apps/web/src/components/editor/panels/assets/views/stickers.tsx index 1ab35d7d..318aaccb 100644 --- a/apps/web/src/components/editor/panels/assets/views/stickers.tsx +++ b/apps/web/src/components/editor/panels/assets/views/stickers.tsx @@ -1,455 +1,455 @@ -"use client"; - -import Image from "next/image"; -import type { CSSProperties } from "react"; -import { useEffect, useState } from "react"; -import { toast } from "sonner"; -import { DraggableItem } from "@/components/editor/panels/assets/draggable-item"; -import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; -import { Spinner } from "@/components/ui/spinner"; -import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { useEditor } from "@/hooks/use-editor"; -import { resolveStickerIntrinsicSize } from "@/lib/stickers"; -import { - buildGraphicElement, - buildStickerElement, -} from "@/lib/timeline/element-utils"; -import { STICKER_CATEGORIES } from "@/lib/stickers/categories"; -import { getRegionLabel, resolveQueryToRegions } from "@/lib/stickers"; -import { parseShapeStickerId } from "@/lib/stickers/providers/shapes"; -import type { TimelineDragData } from "@/lib/timeline/drag"; -import type { - StickerBrowseSection, - StickerCategory, - StickerItem as StickerData, -} from "@/lib/stickers"; -import { useStickersStore } from "@/stores/stickers-store"; -import { cn } from "@/utils/ui"; -import { - HappyIcon, -} from "@hugeicons/core-free-icons"; -import { HugeiconsIcon } from "@hugeicons/react"; - -export function StickersView() { - const { - browseContent, - browseStickers, - searchQuery, - searchStickers, - selectedCategory, - setSearchQuery, - setSelectedCategory, - viewMode, - } = useStickersStore(); - - useEffect(() => { - if (viewMode === "browse" && !browseContent) { - void browseStickers(); - } - }, [browseContent, browseStickers, viewMode]); - - return ( -
-
- { - setSearchQuery({ query: e.target.value }); - void searchStickers({ query: e.target.value }); - }} - showClearIcon - onClear={() => { - setSearchQuery({ query: "" }); - void searchStickers({ query: "" }); - }} - className="w-full" - containerClassName="w-full" - /> -
- - { - setSelectedCategory({ category: value as StickerCategory }); - }} - variant="underline" - className="mt-2 flex min-h-0 flex-1 flex-col" - > - - {Object.entries(STICKER_CATEGORIES).map(([key, label]) => ( - - {label} - - ))} - -
- -
- -
- ); -} - -function StickerGrid({ - items, - shouldCapSize = false, -}: { - items: StickerData[]; - shouldCapSize?: boolean; -}) { - const gridStyle: CSSProperties & { - "--sticker-min": string; - "--sticker-max"?: string; - } = { - gridTemplateColumns: shouldCapSize - ? "repeat(auto-fill, minmax(var(--sticker-min, 80px), var(--sticker-max, 140px)))" - : "repeat(auto-fill, minmax(var(--sticker-min, 80px), 1fr))", - "--sticker-min": "80px", - ...(shouldCapSize ? { "--sticker-max": "140px" } : {}), - }; - - return ( -
- {items.map((item) => ( - - ))} -
- ); -} - -function StickerRow({ items }: { items: StickerData[] }) { - return ( -
- {items.map((item) => ( -
- -
- ))} -
- ); -} - -function EmptyView({ message }: { message: string }) { - return ( -
- -
-

No stickers found

-

{message}

-
-
- ); -} - -function RegionBanner({ region }: { region: string }) { - return ( -
- - {region} -
- ); -} - -function StickersContentView() { - const { - browseContent, - clearRecentStickers, - isBrowsing, - isSearching, - searchQuery, - searchResults, - selectedCategory, - setSelectedCategory, - viewMode, - } = useStickersStore(); - - if (viewMode === "search") { - if (isSearching) { - return ( -
- -
- ); - } - - if (searchResults?.items.length) { - const normalizedQuery = searchQuery.trim().toLowerCase(); - const isRegionSearch = - selectedCategory === "flags" && - resolveQueryToRegions({ query: normalizedQuery }) !== null; - const regionLabel = getRegionLabel({ query: normalizedQuery }); - - return ( -
- {isRegionSearch && } -
- - {searchResults.total} results - -
- -
- ); - } - - // "all" tab search — sections are in browseContent, fall through to section rendering below - if (selectedCategory !== "all" && searchQuery) { - return ; - } - } - - if (isBrowsing && !browseContent) { - return ( -
- -
- ); - } - - if (!browseContent?.sections.length) { - const categoryLabel = STICKER_CATEGORIES[selectedCategory]; - return ( - - ); - } - - return ( -
- {browseContent.sections.map((section) => ( - { - setSelectedCategory({ category }); - }} - /> - ))} -
- ); -} - -function StickerSection({ - section, - onClearRecent, - onSeeAll, -}: { - section: StickerBrowseSection; - onClearRecent: () => void; - onSeeAll: (category: StickerCategory) => void; -}) { - const hasHeader = - Boolean(section.title) || section.id === "recent" || section.action; - - return ( -
- {hasHeader && ( -
- {section.title ? ( -

{section.title}

- ) : ( -
- )} - -
- {section.id === "recent" && ( - - )} - - {section.action?.type === "see-all" && section.action.category && ( - - )} -
-
- )} - - {section.layout === "row" ? ( - - ) : ( - - )} -
- ); -} - -interface StickerItemProps { - item: StickerData; - shouldCapSize?: boolean; - containerClassName?: string; -} - -function StickerItem({ - item, - shouldCapSize = false, - containerClassName, -}: StickerItemProps) { - const editor = useEditor(); - const { addToRecentStickers } = useStickersStore(); - const [isAdding, setIsAdding] = useState(false); - const [hasImageError, setHasImageError] = useState(false); - - useEffect(() => { - if (!item.id) { - return; - } - - setHasImageError(false); - }, [item.id]); - - const displayName = item.name; - const shapePreset = - item.provider === "shapes" ? parseShapeStickerId({ stickerId: item.id }) : null; - - const handleAdd = async () => { - setIsAdding(true); - try { - const currentTime = editor.playback.getCurrentTime(); - - let element: - | ReturnType - | ReturnType; - if (shapePreset) { - element = buildGraphicElement({ - definitionId: shapePreset.definitionId, - name: shapePreset.name, - startTime: currentTime, - params: shapePreset.params, - }); - } else { - const { width: intrinsicWidth, height: intrinsicHeight } = - await resolveStickerIntrinsicSize({ stickerId: item.id }); - element = buildStickerElement({ - stickerId: item.id, - name: item.name, - startTime: currentTime, - intrinsicWidth, - intrinsicHeight, - }); - } - - editor.timeline.insertElement({ - placement: { mode: "auto" }, - element, - }); - - addToRecentStickers({ stickerId: item.id }); - } catch (error) { - console.error("Failed to add sticker:", error); - toast.error("Failed to add sticker to timeline"); - } finally { - setIsAdding(false); - } - }; - - const preview = ( -
- {hasImageError ? ( - - {displayName} - - ) : ( - {displayName} setHasImageError(true)} - loading="lazy" - unoptimized - /> - )} -
- ); - - const dragData: TimelineDragData = shapePreset - ? { - id: item.id, - type: "graphic", - name: displayName, - definitionId: shapePreset.definitionId, - params: shapePreset.params ?? {}, - } - : { - id: item.id, - type: "sticker", - name: displayName, - stickerId: item.id, - }; - - return ( -
- - {isAdding && ( -
- -
- )} -
- ); -} +"use client"; + +import Image from "next/image"; +import type { CSSProperties } from "react"; +import { useEffect, useState } from "react"; +import { toast } from "sonner"; +import { DraggableItem } from "@/components/editor/panels/assets/draggable-item"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Spinner } from "@/components/ui/spinner"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { useEditor } from "@/hooks/use-editor"; +import { resolveStickerIntrinsicSize } from "@/lib/stickers"; +import { + buildGraphicElement, + buildStickerElement, +} from "@/lib/timeline/element-utils"; +import { STICKER_CATEGORIES } from "@/lib/stickers/categories"; +import { getRegionLabel, resolveQueryToRegions } from "@/lib/stickers"; +import { parseShapeStickerId } from "@/lib/stickers/providers/shapes"; +import type { TimelineDragData } from "@/lib/timeline/drag"; +import type { + StickerBrowseSection, + StickerCategory, + StickerItem as StickerData, +} from "@/lib/stickers"; +import { useStickersStore } from "@/stores/stickers-store"; +import { cn } from "@/utils/ui"; +import { + HappyIcon, +} from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; + +export function StickersView() { + const { + browseContent, + browseStickers, + searchQuery, + searchStickers, + selectedCategory, + setSearchQuery, + setSelectedCategory, + viewMode, + } = useStickersStore(); + + useEffect(() => { + if (viewMode === "browse" && !browseContent) { + void browseStickers(); + } + }, [browseContent, browseStickers, viewMode]); + + return ( +
+
+ { + setSearchQuery({ query: e.target.value }); + void searchStickers({ query: e.target.value }); + }} + showClearIcon + onClear={() => { + setSearchQuery({ query: "" }); + void searchStickers({ query: "" }); + }} + className="w-full" + containerClassName="w-full" + /> +
+ + { + setSelectedCategory({ category: value as StickerCategory }); + }} + variant="underline" + className="mt-2 flex min-h-0 flex-1 flex-col" + > + + {Object.entries(STICKER_CATEGORIES).map(([key, label]) => ( + + {label} + + ))} + +
+ +
+ +
+ ); +} + +function StickerGrid({ + items, + shouldCapSize = false, +}: { + items: StickerData[]; + shouldCapSize?: boolean; +}) { + const gridStyle: CSSProperties & { + "--sticker-min": string; + "--sticker-max"?: string; + } = { + gridTemplateColumns: shouldCapSize + ? "repeat(auto-fill, minmax(var(--sticker-min, 80px), var(--sticker-max, 140px)))" + : "repeat(auto-fill, minmax(var(--sticker-min, 80px), 1fr))", + "--sticker-min": "80px", + ...(shouldCapSize ? { "--sticker-max": "140px" } : {}), + }; + + return ( +
+ {items.map((item) => ( + + ))} +
+ ); +} + +function StickerRow({ items }: { items: StickerData[] }) { + return ( +
+ {items.map((item) => ( +
+ +
+ ))} +
+ ); +} + +function EmptyView({ message }: { message: string }) { + return ( +
+ +
+

No stickers found

+

{message}

+
+
+ ); +} + +function RegionBanner({ region }: { region: string }) { + return ( +
+ + {region} +
+ ); +} + +function StickersContentView() { + const { + browseContent, + clearRecentStickers, + isBrowsing, + isSearching, + searchQuery, + searchResults, + selectedCategory, + setSelectedCategory, + viewMode, + } = useStickersStore(); + + if (viewMode === "search") { + if (isSearching) { + return ( +
+ +
+ ); + } + + if (searchResults?.items.length) { + const normalizedQuery = searchQuery.trim().toLowerCase(); + const isRegionSearch = + selectedCategory === "flags" && + resolveQueryToRegions({ query: normalizedQuery }) !== null; + const regionLabel = getRegionLabel({ query: normalizedQuery }); + + return ( +
+ {isRegionSearch && } +
+ + {searchResults.total} results + +
+ +
+ ); + } + + // "all" tab search — sections are in browseContent, fall through to section rendering below + if (selectedCategory !== "all" && searchQuery) { + return ; + } + } + + if (isBrowsing && !browseContent) { + return ( +
+ +
+ ); + } + + if (!browseContent?.sections.length) { + const categoryLabel = STICKER_CATEGORIES[selectedCategory]; + return ( + + ); + } + + return ( +
+ {browseContent.sections.map((section) => ( + { + setSelectedCategory({ category }); + }} + /> + ))} +
+ ); +} + +function StickerSection({ + section, + onClearRecent, + onSeeAll, +}: { + section: StickerBrowseSection; + onClearRecent: () => void; + onSeeAll: (category: StickerCategory) => void; +}) { + const hasHeader = + Boolean(section.title) || section.id === "recent" || section.action; + + return ( +
+ {hasHeader && ( +
+ {section.title ? ( +

{section.title}

+ ) : ( +
+ )} + +
+ {section.id === "recent" && ( + + )} + + {section.action?.type === "see-all" && section.action.category && ( + + )} +
+
+ )} + + {section.layout === "row" ? ( + + ) : ( + + )} +
+ ); +} + +interface StickerItemProps { + item: StickerData; + shouldCapSize?: boolean; + containerClassName?: string; +} + +function StickerItem({ + item, + shouldCapSize = false, + containerClassName, +}: StickerItemProps) { + const editor = useEditor(); + const { addToRecentStickers } = useStickersStore(); + const [isAdding, setIsAdding] = useState(false); + const [hasImageError, setHasImageError] = useState(false); + + useEffect(() => { + if (!item.id) { + return; + } + + setHasImageError(false); + }, [item.id]); + + const displayName = item.name; + const shapePreset = + item.provider === "shapes" ? parseShapeStickerId({ stickerId: item.id }) : null; + + const handleAdd = async () => { + setIsAdding(true); + try { + const currentTime = editor.playback.getCurrentTime(); + + let element: + | ReturnType + | ReturnType; + if (shapePreset) { + element = buildGraphicElement({ + definitionId: shapePreset.definitionId, + name: shapePreset.name, + startTime: currentTime, + params: shapePreset.params, + }); + } else { + const { width: intrinsicWidth, height: intrinsicHeight } = + await resolveStickerIntrinsicSize({ stickerId: item.id }); + element = buildStickerElement({ + stickerId: item.id, + name: item.name, + startTime: currentTime, + intrinsicWidth, + intrinsicHeight, + }); + } + + editor.timeline.insertElement({ + placement: { mode: "auto" }, + element, + }); + + addToRecentStickers({ stickerId: item.id }); + } catch (error) { + console.error("Failed to add sticker:", error); + toast.error("Failed to add sticker to timeline"); + } finally { + setIsAdding(false); + } + }; + + const preview = ( +
+ {hasImageError ? ( + + {displayName} + + ) : ( + {displayName} setHasImageError(true)} + loading="lazy" + unoptimized + /> + )} +
+ ); + + const dragData: TimelineDragData = shapePreset + ? { + id: item.id, + type: "graphic", + name: displayName, + definitionId: shapePreset.definitionId, + params: shapePreset.params ?? {}, + } + : { + id: item.id, + type: "sticker", + name: displayName, + stickerId: item.id, + }; + + return ( +
+ + {isAdding && ( +
+ +
+ )} +
+ ); +} diff --git a/apps/web/src/components/ui/button.tsx b/apps/web/src/components/ui/button.tsx index 677d9330..21b2c434 100644 --- a/apps/web/src/components/ui/button.tsx +++ b/apps/web/src/components/ui/button.tsx @@ -14,6 +14,7 @@ const buttonVariants = cva( "bg-destructive text-destructive-foreground hover:bg-destructive/80", "destructive-foreground": "border bg-background hover:bg-destructive/15 text-destructive", + caution: "text-caution hover:bg-caution/10", outline: "border border-border bg-background hover:bg-accent", secondary: "bg-secondary text-secondary-foreground border border-secondary-border", diff --git a/apps/web/src/core/index.ts b/apps/web/src/core/index.ts index e93b937d..483a26c1 100644 --- a/apps/web/src/core/index.ts +++ b/apps/web/src/core/index.ts @@ -9,8 +9,10 @@ import { SaveManager } from "./managers/save-manager"; import { AudioManager } from "./managers/audio-manager"; import { SelectionManager } from "./managers/selection-manager"; import { ClipboardManager } from "./managers/clipboard-manager"; +import { DiagnosticsManager } from "./managers/diagnostics-manager"; import { registerDefaultEffects } from "@/lib/effects"; import { registerDefaultMasks } from "@/lib/masks"; +import { registerTranscriptionDiagnostics } from "@/lib/transcription/diagnostics"; export class EditorCore { private static instance: EditorCore | null = null; @@ -25,6 +27,7 @@ export class EditorCore { public readonly audio: AudioManager; public readonly selection: SelectionManager; public readonly clipboard: ClipboardManager; + public readonly diagnostics: DiagnosticsManager; private constructor() { registerDefaultEffects(); @@ -40,6 +43,8 @@ export class EditorCore { this.audio = new AudioManager(this); this.selection = new SelectionManager(this); this.clipboard = new ClipboardManager(this); + this.diagnostics = new DiagnosticsManager(this); + registerTranscriptionDiagnostics({ diagnostics: this.diagnostics }); this.playback.bindTimelineScope(); this.command.registerReactor(() => { const activeScene = this.scenes.getActiveSceneOrNull(); diff --git a/apps/web/src/core/managers/diagnostics-manager.ts b/apps/web/src/core/managers/diagnostics-manager.ts new file mode 100644 index 00000000..d022a350 --- /dev/null +++ b/apps/web/src/core/managers/diagnostics-manager.ts @@ -0,0 +1,38 @@ +import type { EditorCore } from "@/core"; +import type { DiagnosticDefinition } from "@/lib/diagnostics/types"; + +interface DiagnosticRegistration extends DiagnosticDefinition { + check: (editor: EditorCore) => boolean; +} + +export class DiagnosticsManager { + private readonly registrations: DiagnosticRegistration[] = []; + private readonly listeners = new Set<() => void>(); + + constructor(private editor: EditorCore) {} + + register(registration: DiagnosticRegistration): void { + this.registrations.push(registration); + this.notify(); + } + + getActive(options?: { scope?: string }): ReadonlyArray { + const candidates = + options?.scope !== undefined + ? this.registrations.filter((r) => r.scope === options.scope) + : this.registrations; + + return candidates.filter((r) => r.check(this.editor)); + } + + subscribe(listener: () => void): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + notify(): void { + this.listeners.forEach((listener) => { + listener(); + }); + } +} diff --git a/apps/web/src/hooks/use-editor.ts b/apps/web/src/hooks/use-editor.ts index 1a5ca8ae..ce48e036 100644 --- a/apps/web/src/hooks/use-editor.ts +++ b/apps/web/src/hooks/use-editor.ts @@ -32,6 +32,7 @@ export function useEditor( editor.renderer.subscribe(onChange), editor.selection.subscribe(onChange), editor.clipboard.subscribe(onChange), + editor.diagnostics.subscribe(onChange), ]; return () => { unsubscribers.forEach((unsubscribe) => { diff --git a/apps/web/src/lib/diagnostics/types.ts b/apps/web/src/lib/diagnostics/types.ts new file mode 100644 index 00000000..486f362c --- /dev/null +++ b/apps/web/src/lib/diagnostics/types.ts @@ -0,0 +1,8 @@ +export type DiagnosticSeverity = "caution" | "error"; + +export interface DiagnosticDefinition { + id: string; + scope: string; + severity: DiagnosticSeverity; + message: string; +} diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.ts index 5a45b6ea..04ea580f 100644 --- a/apps/web/src/lib/media/audio.ts +++ b/apps/web/src/lib/media/audio.ts @@ -1,5 +1,6 @@ import type { AudioElement, + VideoElement, LibraryAudioElement, RetimeConfig, SceneTracks, @@ -12,9 +13,7 @@ import { hasAnimatedVolume, resolveEffectiveAudioGain, } from "@/lib/timeline/audio-state"; -import { - doesElementHaveEnabledAudio, -} from "@/lib/timeline/audio-separation"; +import { doesElementHaveEnabledAudio } from "@/lib/timeline/audio-separation"; import { canElementHaveAudio, hasMediaId } from "@/lib/timeline/element-utils"; import { canTrackHaveAudio } from "@/lib/timeline"; import { mediaSupportsAudio } from "@/lib/media/media-utils"; @@ -83,22 +82,23 @@ export async function decodeAudioToFloat32({ return { samples, sampleRate: audioBuffer.sampleRate }; } -export async function collectAudioElements({ +export interface AudibleElementCandidate { + element: AudioElement | VideoElement; + mediaAsset: MediaAsset | null; +} + +export function collectAudibleCandidates({ tracks, mediaAssets, - audioContext, }: { tracks: SceneTracks; mediaAssets: MediaAsset[]; - audioContext: AudioContext; -}): Promise { - const orderedTracks = [...tracks.overlay, tracks.main, ...tracks.audio]; - const mediaMap = new Map( - mediaAssets.map((media) => [media.id, media]), - ); - const pendingElements: Array> = []; +}): AudibleElementCandidate[] { + const allTracks = [...tracks.overlay, tracks.main, ...tracks.audio]; + const mediaMap = new Map(mediaAssets.map((a) => [a.id, a])); + const candidates: AudibleElementCandidate[] = []; - for (const track of orderedTracks) { + for (const track of allTracks) { if (canTrackHaveAudio(track) && track.muted) continue; for (const element of track.elements) { @@ -110,65 +110,95 @@ export async function collectAudioElements({ : null; if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; - const isTrackMuted = canTrackHaveAudio(track) && track.muted; + candidates.push({ element, mediaAsset }); + } + } - if (element.type === "audio") { - pendingElements.push( - resolveAudioBufferForElement({ - element, - mediaMap, - audioContext, - }).then((audioBuffer) => { - if (!audioBuffer) return null; - const muted = element.muted === true || isTrackMuted; - return { - timelineElement: element, - buffer: audioBuffer, - startTime: element.startTime / TICKS_PER_SECOND, - duration: element.duration / TICKS_PER_SECOND, - trimStart: element.trimStart / TICKS_PER_SECOND, - trimEnd: element.trimEnd / TICKS_PER_SECOND, - volume: resolveEffectiveAudioGain({ - element, - trackMuted: isTrackMuted, - localTime: 0, - }), - muted, - retime: element.retime, - }; - }), - ); - continue; - } + return candidates; +} - if (element.type === "video") { - if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue; +export function timelineHasAudio({ + tracks, + mediaAssets, +}: { + tracks: SceneTracks; + mediaAssets: MediaAsset[]; +}): boolean { + return collectAudibleCandidates({ tracks, mediaAssets }).some( + ({ element }) => element.muted !== true, + ); +} - pendingElements.push( - resolveAudioBufferForVideoElement({ - mediaAsset, - audioContext, - }).then((audioBuffer) => { - if (!audioBuffer) return null; - const muted = (element.muted ?? false) || isTrackMuted; - return { - timelineElement: element, - buffer: audioBuffer, - startTime: element.startTime / TICKS_PER_SECOND, - duration: element.duration / TICKS_PER_SECOND, - trimStart: element.trimStart / TICKS_PER_SECOND, - trimEnd: element.trimEnd / TICKS_PER_SECOND, - volume: resolveEffectiveAudioGain({ - element, - trackMuted: isTrackMuted, - localTime: 0, - }), - muted, - retime: element.retime, - }; - }), - ); - } +export async function collectAudioElements({ + tracks, + mediaAssets, + audioContext, +}: { + tracks: SceneTracks; + mediaAssets: MediaAsset[]; + audioContext: AudioContext; +}): Promise { + const candidates = collectAudibleCandidates({ tracks, mediaAssets }); + const mediaMap = new Map( + mediaAssets.map((media) => [media.id, media]), + ); + const pendingElements: Array> = []; + + for (const { element, mediaAsset } of candidates) { + if (element.type === "audio") { + pendingElements.push( + resolveAudioBufferForElement({ + element, + mediaMap, + audioContext, + }).then((audioBuffer) => { + if (!audioBuffer) return null; + return { + timelineElement: element, + buffer: audioBuffer, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, + volume: resolveEffectiveAudioGain({ + element, + trackMuted: false, + localTime: 0, + }), + muted: element.muted === true, + retime: element.retime, + }; + }), + ); + continue; + } + + if (element.type === "video") { + if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue; + + pendingElements.push( + resolveAudioBufferForVideoElement({ + mediaAsset, + audioContext, + }).then((audioBuffer) => { + if (!audioBuffer) return null; + return { + timelineElement: element, + buffer: audioBuffer, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, + volume: resolveEffectiveAudioGain({ + element, + trackMuted: false, + localTime: 0, + }), + muted: element.muted ?? false, + retime: element.retime, + }; + }), + ); } } @@ -339,16 +369,16 @@ async function fetchLibraryAudioSource({ type: "audio/mpeg", }); - return { - timelineElement: element, - file, - startTime: element.startTime / TICKS_PER_SECOND, - duration: element.duration / TICKS_PER_SECOND, - trimStart: element.trimStart / TICKS_PER_SECOND, - trimEnd: element.trimEnd / TICKS_PER_SECOND, - volume, - retime: element.retime, - }; + return { + timelineElement: element, + file, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, + volume, + retime: element.retime, + }; } catch (error) { console.warn("Failed to fetch library audio:", error); return null; diff --git a/apps/web/src/lib/media/mediabunny.ts b/apps/web/src/lib/media/mediabunny.ts index a313c6f0..1e97fc42 100644 --- a/apps/web/src/lib/media/mediabunny.ts +++ b/apps/web/src/lib/media/mediabunny.ts @@ -1,158 +1,168 @@ -import { Input, ALL_FORMATS, BlobSource } from "mediabunny"; -import { createTimelineAudioBuffer } from "@/lib/media/audio"; -import type { SceneTracks } from "@/lib/timeline"; -import type { MediaAsset } from "@/lib/media/types"; - -export async function getVideoInfo({ - videoFile, -}: { - videoFile: File; -}): Promise<{ - duration: number; - width: number; - height: number; - fps: number; - hasAudio: boolean; -}> { - const input = new Input({ - source: new BlobSource(videoFile), - formats: ALL_FORMATS, - }); - - const duration = await input.computeDuration(); - const videoTrack = await input.getPrimaryVideoTrack(); - - if (!videoTrack) { - throw new Error("No video track found in the file"); - } - - const packetStats = await videoTrack.computePacketStats(100); - const fps = packetStats.averagePacketRate; - const audioTrack = await input.getPrimaryAudioTrack(); - - return { - duration, - width: videoTrack.displayWidth, - height: videoTrack.displayHeight, - fps, - hasAudio: audioTrack !== null, - }; -} - -const SAMPLE_RATE = 44100; -const NUM_CHANNELS = 2; - -export const extractTimelineAudio = async ({ - tracks, - mediaAssets, - totalDuration, - onProgress, -}: { - tracks: SceneTracks; - mediaAssets: MediaAsset[]; - totalDuration: number; - onProgress?: (progress: number) => void; -}): Promise => { - if (totalDuration === 0) { - return createWavBlob({ samples: new Float32Array(SAMPLE_RATE * 0.1) }); - } - - onProgress?.(10); - - const audioBuffer = await createTimelineAudioBuffer({ - tracks, - mediaAssets, - duration: totalDuration, - sampleRate: SAMPLE_RATE, - }); - - if (!audioBuffer) { - const silentDuration = Math.max(1, totalDuration); - const silentSamples = new Float32Array( - Math.ceil(silentDuration * SAMPLE_RATE) * NUM_CHANNELS, - ); - return createWavBlob({ samples: silentSamples }); - } - - onProgress?.(90); - - const interleavedSamples = interleaveAudioBuffer({ audioBuffer }); - onProgress?.(100); - - return createWavBlob({ samples: interleavedSamples }); -}; - -function interleaveAudioBuffer({ - audioBuffer, -}: { - audioBuffer: AudioBuffer; -}): Float32Array { - const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels); - const interleavedSamples = new Float32Array( - audioBuffer.length * NUM_CHANNELS, - ); - - for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) { - for (let channel = 0; channel < NUM_CHANNELS; channel++) { - const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1)); - interleavedSamples[sampleIndex * NUM_CHANNELS + channel] = - audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0; - } - } - - return interleavedSamples; -} - -function createWavBlob({ samples }: { samples: Float32Array }): Blob { - const numChannels = NUM_CHANNELS; - const bitsPerSample = 16; - const bytesPerSample = bitsPerSample / 8; - const numSamples = samples.length / numChannels; - const dataSize = numSamples * numChannels * bytesPerSample; - const buffer = new ArrayBuffer(44 + dataSize); - const view = new DataView(buffer); - - // riff header - writeString({ view, offset: 0, str: "RIFF" }); - view.setUint32(4, 36 + dataSize, true); - writeString({ view, offset: 8, str: "WAVE" }); - - // fmt chunk - writeString({ view, offset: 12, str: "fmt " }); - view.setUint32(16, 16, true); - view.setUint16(20, 1, true); - view.setUint16(22, numChannels, true); - view.setUint32(24, SAMPLE_RATE, true); - view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true); - view.setUint16(32, numChannels * bytesPerSample, true); - view.setUint16(34, bitsPerSample, true); - - // data chunk - writeString({ view, offset: 36, str: "data" }); - view.setUint32(40, dataSize, true); - - // convert float32 to int16 and write - let offset = 44; - for (let i = 0; i < samples.length; i++) { - const sample = Math.max(-1, Math.min(1, samples[i])); - const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff; - view.setInt16(offset, int16, true); - offset += 2; - } - - return new Blob([buffer], { type: "audio/wav" }); -} - -function writeString({ - view, - offset, - str, -}: { - view: DataView; - offset: number; - str: string; -}): void { - for (let i = 0; i < str.length; i++) { - view.setUint8(offset + i, str.charCodeAt(i)); - } -} +import { Input, ALL_FORMATS, BlobSource } from "mediabunny"; +import { createTimelineAudioBuffer } from "@/lib/media/audio"; +import type { SceneTracks } from "@/lib/timeline"; +import type { MediaAsset } from "@/lib/media/types"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; + +export async function getVideoInfo({ + videoFile, +}: { + videoFile: File; +}): Promise<{ + duration: number; + width: number; + height: number; + fps: number; + hasAudio: boolean; +}> { + const input = new Input({ + source: new BlobSource(videoFile), + formats: ALL_FORMATS, + }); + + const duration = await input.computeDuration(); + const videoTrack = await input.getPrimaryVideoTrack(); + + if (!videoTrack) { + throw new Error("No video track found in the file"); + } + + const packetStats = await videoTrack.computePacketStats(100); + const fps = packetStats.averagePacketRate; + const audioTrack = await input.getPrimaryAudioTrack(); + + return { + duration, + width: videoTrack.displayWidth, + height: videoTrack.displayHeight, + fps, + hasAudio: audioTrack !== null, + }; +} + +const SAMPLE_RATE = 44100; +const NUM_CHANNELS = 2; +const EMPTY_TIMELINE_SILENT_DURATION_SECONDS = 0.1; +const MIN_SILENT_DURATION_SECONDS = 0.001; + +export const extractTimelineAudio = async ({ + tracks, + mediaAssets, + totalDuration, + onProgress, +}: { + tracks: SceneTracks; + mediaAssets: MediaAsset[]; + totalDuration: number; + onProgress?: (progress: number) => void; +}): Promise => { + if (totalDuration === 0) { + return createWavBlob({ + samples: new Float32Array( + SAMPLE_RATE * EMPTY_TIMELINE_SILENT_DURATION_SECONDS, + ), + }); + } + + onProgress?.(10); + + const audioBuffer = await createTimelineAudioBuffer({ + tracks, + mediaAssets, + duration: totalDuration, + sampleRate: SAMPLE_RATE, + }); + + if (!audioBuffer) { + const silentDurationSeconds = Math.max( + MIN_SILENT_DURATION_SECONDS, + totalDuration / TICKS_PER_SECOND, + ); + const silentSamples = new Float32Array( + Math.ceil(silentDurationSeconds * SAMPLE_RATE) * NUM_CHANNELS, + ); + return createWavBlob({ samples: silentSamples }); + } + + onProgress?.(90); + + const interleavedSamples = interleaveAudioBuffer({ audioBuffer }); + onProgress?.(100); + + return createWavBlob({ samples: interleavedSamples }); +}; + +function interleaveAudioBuffer({ + audioBuffer, +}: { + audioBuffer: AudioBuffer; +}): Float32Array { + const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels); + const interleavedSamples = new Float32Array( + audioBuffer.length * NUM_CHANNELS, + ); + + for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) { + for (let channel = 0; channel < NUM_CHANNELS; channel++) { + const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1)); + interleavedSamples[sampleIndex * NUM_CHANNELS + channel] = + audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0; + } + } + + return interleavedSamples; +} + +function createWavBlob({ samples }: { samples: Float32Array }): Blob { + const numChannels = NUM_CHANNELS; + const bitsPerSample = 16; + const bytesPerSample = bitsPerSample / 8; + const numSamples = samples.length / numChannels; + const dataSize = numSamples * numChannels * bytesPerSample; + const buffer = new ArrayBuffer(44 + dataSize); + const view = new DataView(buffer); + + // riff header + writeString({ view, offset: 0, str: "RIFF" }); + view.setUint32(4, 36 + dataSize, true); + writeString({ view, offset: 8, str: "WAVE" }); + + // fmt chunk + writeString({ view, offset: 12, str: "fmt " }); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, numChannels, true); + view.setUint32(24, SAMPLE_RATE, true); + view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true); + view.setUint16(32, numChannels * bytesPerSample, true); + view.setUint16(34, bitsPerSample, true); + + // data chunk + writeString({ view, offset: 36, str: "data" }); + view.setUint32(40, dataSize, true); + + // convert float32 to int16 and write + let offset = 44; + for (let i = 0; i < samples.length; i++) { + const sample = Math.max(-1, Math.min(1, samples[i])); + const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff; + view.setInt16(offset, int16, true); + offset += 2; + } + + return new Blob([buffer], { type: "audio/wav" }); +} + +function writeString({ + view, + offset, + str, +}: { + view: DataView; + offset: number; + str: string; +}): void { + for (let i = 0; i < str.length; i++) { + view.setUint8(offset + i, str.charCodeAt(i)); + } +} diff --git a/apps/web/src/lib/timeline/audio-separation/index.ts b/apps/web/src/lib/timeline/audio-separation/index.ts index fe3c1ca4..849dbda3 100644 --- a/apps/web/src/lib/timeline/audio-separation/index.ts +++ b/apps/web/src/lib/timeline/audio-separation/index.ts @@ -49,7 +49,9 @@ export function canToggleSourceAudio( element: TimelineElement, mediaAsset: MediaAudioState | null | undefined, ): element is VideoElement { - return canRecoverSourceAudio(element) || canExtractSourceAudio(element, mediaAsset); + return ( + canRecoverSourceAudio(element) || canExtractSourceAudio(element, mediaAsset) + ); } export function doesElementHaveEnabledAudio({ @@ -63,7 +65,11 @@ export function doesElementHaveEnabledAudio({ return true; } - return !!mediaAsset && mediaAsset.hasAudio !== false && isSourceAudioEnabled({ element }); + return ( + !!mediaAsset && + mediaAsset.hasAudio !== false && + isSourceAudioEnabled({ element }) + ); } export function buildSeparatedAudioElement({ @@ -100,7 +106,9 @@ export function getSourceAudioActionLabel({ }: { element: VideoElement; }): "Extract audio" | "Recover audio" { - return isSourceAudioSeparated({ element }) ? "Recover audio" : "Extract audio"; + return isSourceAudioSeparated({ element }) + ? "Recover audio" + : "Extract audio"; } function cloneVolumeAnimations({ diff --git a/apps/web/src/lib/transcription/diagnostics.ts b/apps/web/src/lib/transcription/diagnostics.ts new file mode 100644 index 00000000..a1921699 --- /dev/null +++ b/apps/web/src/lib/transcription/diagnostics.ts @@ -0,0 +1,25 @@ +import type { DiagnosticsManager } from "@/core/managers/diagnostics-manager"; +import { timelineHasAudio } from "@/lib/media/audio"; + +export const TRANSCRIPTION_DIAGNOSTICS_SCOPE = "transcription"; + +export function registerTranscriptionDiagnostics({ + diagnostics, +}: { + diagnostics: DiagnosticsManager; +}): void { + diagnostics.register({ + id: "transcription.no_audio", + scope: TRANSCRIPTION_DIAGNOSTICS_SCOPE, + severity: "caution", + message: "No audio detected. Add a clip with audio to the timeline first.", + check: (editor) => { + const scene = editor.scenes.getActiveSceneOrNull(); + if (!scene) return false; + return !timelineHasAudio({ + tracks: scene.tracks, + mediaAssets: editor.media.getAssets(), + }); + }, + }); +}