@@ -439,6 +448,29 @@ function ElementContent({
);
}
+ if (mediaAsset.type === "video") {
+ const trackHeight = getTrackHeight({ type: track.type });
+ const trimStart = element.trimStart ?? 0;
+
+ return (
+
+ );
+ }
+
return (
{element.name}
);
@@ -446,7 +478,10 @@ function ElementContent({
function CopyMenuItem() {
return (
-
}>
+
}
+ >
Copy
);
@@ -502,7 +537,10 @@ function VisibilityMenuItem({
};
return (
-
+
{isHidden ? "Show" : "Hide"}
);
diff --git a/apps/web/src/core/managers/media-manager.ts b/apps/web/src/core/managers/media-manager.ts
index fd8726d9..1a27eff8 100644
--- a/apps/web/src/core/managers/media-manager.ts
+++ b/apps/web/src/core/managers/media-manager.ts
@@ -3,6 +3,7 @@ import type { MediaAsset } from "@/types/assets";
import { storageService } from "@/services/storage/service";
import { generateUUID } from "@/utils/id";
import { videoCache } from "@/services/video-cache/service";
+import { filmstripService } from "@/services/filmstrip/service";
import { hasMediaId } from "@/lib/timeline/element-utils";
export class MediaManager {
@@ -46,6 +47,7 @@ export class MediaManager {
const asset = this.assets.find((asset) => asset.id === id);
videoCache.clearVideo({ mediaId: id });
+ filmstripService.clearMedia({ mediaId: id });
if (asset?.url) {
URL.revokeObjectURL(asset.url);
@@ -124,6 +126,7 @@ export class MediaManager {
clearAllAssets(): void {
videoCache.clearAll();
+ filmstripService.destroy();
this.assets.forEach((asset) => {
if (asset.url) {
diff --git a/apps/web/src/hooks/timeline/use-filmstrip.ts b/apps/web/src/hooks/timeline/use-filmstrip.ts
new file mode 100644
index 00000000..6105845c
--- /dev/null
+++ b/apps/web/src/hooks/timeline/use-filmstrip.ts
@@ -0,0 +1,146 @@
+import { useEffect, useState, useRef } from "react";
+import { filmstripService } from "@/services/filmstrip/service";
+import type {
+ FilmstripFrame,
+ FilmstripStatus,
+} from "@/services/filmstrip/types";
+import {
+ getTierForZoom,
+ getTimestampsForRange,
+} from "@/lib/timeline/filmstrip-utils";
+
+interface UseFilmstripOptions {
+ mediaId: string;
+ file: File | null;
+ duration: number;
+ visibleStartTime: number;
+ visibleEndTime: number;
+ zoomLevel: number;
+}
+
+interface UseFilmstripReturn {
+ frames: FilmstripFrame[];
+ status: FilmstripStatus;
+ progress: number;
+}
+
+export function useFilmstrip({
+ mediaId,
+ file,
+ duration,
+ visibleStartTime,
+ visibleEndTime,
+ zoomLevel,
+}: UseFilmstripOptions): UseFilmstripReturn {
+ const [frames, setFrames] = useState([]);
+ const [status, setStatus] = useState("idle");
+ const [progress, setProgress] = useState(0);
+
+ const tierRef = useRef(0);
+ const requestedTimestampsRef = useRef>(new Set());
+
+ useEffect(() => {
+ if (!file || duration <= 0) {
+ setFrames([]);
+ setStatus("idle");
+ setProgress(0);
+ return;
+ }
+
+ const tier = getTierForZoom({ zoomLevel });
+ tierRef.current = tier;
+
+ const timestamps = getTimestampsForRange({
+ startTime: visibleStartTime,
+ endTime: visibleEndTime,
+ tier: tier as 0 | 1 | 2 | 3,
+ });
+
+ const cachedFrames = filmstripService.getFrames({
+ mediaId,
+ tier,
+ startTime: visibleStartTime,
+ endTime: visibleEndTime,
+ });
+
+ setFrames(cachedFrames);
+
+ const missingTimestamps = timestamps.filter(
+ (ts) => !cachedFrames.some((f) => f.timestamp === ts),
+ );
+
+ if (missingTimestamps.length === 0) {
+ setStatus("ready");
+ setProgress(100);
+ return;
+ }
+
+ setStatus("generating");
+ setProgress((cachedFrames.length / timestamps.length) * 100);
+
+ const newTimestamps = missingTimestamps.filter(
+ (ts) => !requestedTimestampsRef.current.has(ts),
+ );
+
+ if (newTimestamps.length > 0) {
+ for (const ts of newTimestamps) {
+ requestedTimestampsRef.current.add(ts);
+ }
+
+ filmstripService.requestFrames({
+ mediaId,
+ file,
+ timestamps: newTimestamps,
+ tier,
+ });
+ }
+
+ const handleFrame = ({
+ mediaId: frameMediaId,
+ }: {
+ mediaId: string;
+ timestamp: number;
+ dataUrl: string;
+ }) => {
+ if (frameMediaId !== mediaId) {
+ return;
+ }
+
+ if (tierRef.current !== tier) {
+ return;
+ }
+
+ const updatedFrames = filmstripService.getFrames({
+ mediaId,
+ tier,
+ startTime: visibleStartTime,
+ endTime: visibleEndTime,
+ });
+
+ setFrames(updatedFrames);
+ setProgress((updatedFrames.length / timestamps.length) * 100);
+
+ if (updatedFrames.length === timestamps.length) {
+ setStatus("ready");
+ setProgress(100);
+ }
+ };
+
+ const handleError = (errorMediaId: string, _error: string) => {
+ if (errorMediaId === mediaId) {
+ setStatus("error");
+ }
+ };
+
+ filmstripService.on("frame", handleFrame);
+ filmstripService.on("error", handleError);
+
+ return () => {
+ filmstripService.off("frame", handleFrame);
+ filmstripService.off("error", handleError);
+ filmstripService.cancelPending({ mediaId });
+ };
+ }, [mediaId, file, duration, visibleStartTime, visibleEndTime, zoomLevel]);
+
+ return { frames, status, progress };
+}
diff --git a/apps/web/src/lib/timeline/filmstrip-utils.ts b/apps/web/src/lib/timeline/filmstrip-utils.ts
new file mode 100644
index 00000000..8969bc53
--- /dev/null
+++ b/apps/web/src/lib/timeline/filmstrip-utils.ts
@@ -0,0 +1,61 @@
+export type FilmstripTier = 0 | 1 | 2 | 3;
+
+const TIER_INTERVALS: Record = {
+ 0: 5,
+ 1: 1,
+ 2: 0.5,
+ 3: 0.25,
+} as const;
+
+export function getTierForZoom({
+ zoomLevel,
+}: {
+ zoomLevel: number;
+}): FilmstripTier {
+ if (zoomLevel < 0.5) {
+ return 0;
+ }
+ if (zoomLevel < 2) {
+ return 1;
+ }
+ if (zoomLevel < 5) {
+ return 2;
+ }
+ return 3;
+}
+
+export function getIntervalForTier({ tier }: { tier: FilmstripTier }): number {
+ return TIER_INTERVALS[tier];
+}
+
+export function getTimestampsForRange({
+ startTime,
+ endTime,
+ tier,
+}: {
+ startTime: number;
+ endTime: number;
+ tier: FilmstripTier;
+}): number[] {
+ const interval = getIntervalForTier({ tier });
+ const timestamps: number[] = [];
+
+ let current = Math.floor(startTime / interval) * interval;
+
+ while (current <= endTime) {
+ if (current >= startTime) {
+ timestamps.push(current);
+ }
+ current += interval;
+ }
+
+ return timestamps;
+}
+
+export function getThumbnailWidth({
+ trackHeight,
+}: {
+ trackHeight: number;
+}): number {
+ return Math.round(trackHeight * (16 / 9));
+}
diff --git a/apps/web/src/services/filmstrip/service.ts b/apps/web/src/services/filmstrip/service.ts
new file mode 100644
index 00000000..7d5bd3c4
--- /dev/null
+++ b/apps/web/src/services/filmstrip/service.ts
@@ -0,0 +1,250 @@
+import EventEmitter from "eventemitter3";
+import type { FilmstripFrame, FilmstripGenerationProgress } from "./types";
+import { getTimestampsForRange } from "@/lib/timeline/filmstrip-utils";
+
+const MAX_CACHE_SIZE = 150;
+
+type CacheKey = string;
+
+function createCacheKey({
+ mediaId,
+ timestamp,
+ tier,
+}: {
+ mediaId: string;
+ timestamp: number;
+ tier: number;
+}): CacheKey {
+ return `${mediaId}:${tier}:${timestamp}`;
+}
+
+interface LRUCacheEntry {
+ key: CacheKey;
+ frame: FilmstripFrame;
+}
+
+export type FilmstripServiceEvents = {
+ frame: [FilmstripGenerationProgress];
+ error: [string, string];
+};
+
+export class FilmstripService extends EventEmitter {
+ private cache = new Map();
+ private cacheOrder: CacheKey[] = [];
+ private worker: Worker | null = null;
+ private pendingRequests = new Map>();
+
+ private getWorker(): Worker {
+ if (!this.worker) {
+ this.worker = new Worker(new URL("./worker.ts", import.meta.url), {
+ type: "module",
+ });
+
+ this.worker.onmessage = (event) => {
+ const response = event.data;
+
+ if (response.type === "frame") {
+ const pending = this.pendingRequests.get(response.mediaId);
+ const tier = pending?.get(response.timestamp);
+ if (tier !== undefined) {
+ this.handleFrame({
+ mediaId: response.mediaId,
+ timestamp: response.timestamp,
+ dataUrl: response.dataUrl,
+ tier,
+ });
+ }
+ } else if (response.type === "error") {
+ this.emit("error", response.mediaId, response.error);
+ } else if (response.type === "cancelled") {
+ this.pendingRequests.delete(response.mediaId);
+ }
+ };
+ }
+
+ return this.worker;
+ }
+
+ private handleFrame({
+ mediaId,
+ timestamp,
+ dataUrl,
+ tier,
+ }: {
+ mediaId: string;
+ timestamp: number;
+ dataUrl: string;
+ tier: number;
+ }): void {
+ const pending = this.pendingRequests.get(mediaId);
+ if (!pending || !pending.has(timestamp)) {
+ return;
+ }
+
+ pending.delete(timestamp);
+ if (pending.size === 0) {
+ this.pendingRequests.delete(mediaId);
+ }
+
+ const frame: FilmstripFrame = {
+ mediaId,
+ timestamp,
+ tier,
+ dataUrl,
+ };
+
+ this.setFrame({ frame });
+ this.emit("frame", { mediaId, timestamp, dataUrl });
+ }
+
+ private setFrame({ frame }: { frame: FilmstripFrame }): void {
+ const key = createCacheKey({
+ mediaId: frame.mediaId,
+ timestamp: frame.timestamp,
+ tier: frame.tier,
+ });
+
+ if (this.cache.has(key)) {
+ this.moveToEnd({ key });
+ return;
+ }
+
+ if (this.cache.size >= MAX_CACHE_SIZE) {
+ this.evictOldest();
+ }
+
+ this.cache.set(key, { key, frame });
+ this.cacheOrder.push(key);
+ }
+
+ private moveToEnd({ key }: { key: CacheKey }): void {
+ const index = this.cacheOrder.indexOf(key);
+ if (index !== -1) {
+ this.cacheOrder.splice(index, 1);
+ this.cacheOrder.push(key);
+ }
+ }
+
+ private evictOldest(): void {
+ const oldestKey = this.cacheOrder.shift();
+ if (oldestKey) {
+ this.cache.delete(oldestKey);
+ }
+ }
+
+ getFrames({
+ mediaId,
+ tier,
+ startTime,
+ endTime,
+ }: {
+ mediaId: string;
+ tier: number;
+ startTime: number;
+ endTime: number;
+ }): FilmstripFrame[] {
+ const timestamps = getTimestampsForRange({
+ startTime,
+ endTime,
+ tier: tier as 0 | 1 | 2 | 3,
+ });
+
+ const frames: FilmstripFrame[] = [];
+
+ for (const timestamp of timestamps) {
+ const key = createCacheKey({ mediaId, timestamp, tier });
+ const entry = this.cache.get(key);
+
+ if (entry) {
+ this.moveToEnd({ key });
+ frames.push(entry.frame);
+ }
+ }
+
+ return frames;
+ }
+
+ requestFrames({
+ mediaId,
+ file,
+ timestamps,
+ tier,
+ }: {
+ mediaId: string;
+ file: File;
+ timestamps: number[];
+ tier: number;
+ }): void {
+ if (timestamps.length === 0) {
+ return;
+ }
+
+ const pending = this.pendingRequests.get(mediaId) ?? new Map();
+ const newTimestamps = timestamps.filter((ts) => !pending.has(ts));
+
+ if (newTimestamps.length === 0) {
+ return;
+ }
+
+ for (const timestamp of newTimestamps) {
+ pending.set(timestamp, tier);
+ }
+
+ this.pendingRequests.set(mediaId, pending);
+
+ const worker = this.getWorker();
+ worker.postMessage({
+ type: "extract",
+ request: {
+ mediaId,
+ file,
+ timestamps: newTimestamps,
+ },
+ });
+ }
+
+ clearMedia({ mediaId }: { mediaId: string }): void {
+ const keysToDelete: CacheKey[] = [];
+
+ for (const [key, entry] of this.cache) {
+ if (entry.frame.mediaId === mediaId) {
+ keysToDelete.push(key);
+ }
+ }
+
+ for (const key of keysToDelete) {
+ this.cache.delete(key);
+ const index = this.cacheOrder.indexOf(key);
+ if (index !== -1) {
+ this.cacheOrder.splice(index, 1);
+ }
+ }
+
+ this.cancelPending({ mediaId });
+ }
+
+ cancelPending({ mediaId }: { mediaId: string }): void {
+ const pending = this.pendingRequests.get(mediaId);
+ if (pending) {
+ this.pendingRequests.delete(mediaId);
+
+ const worker = this.getWorker();
+ worker.postMessage({
+ type: "cancel",
+ mediaId,
+ });
+ }
+ }
+
+ destroy(): void {
+ if (this.worker) {
+ this.worker.terminate();
+ this.worker = null;
+ }
+ this.cache.clear();
+ this.cacheOrder = [];
+ this.pendingRequests.clear();
+ }
+}
+
+export const filmstripService = new FilmstripService();
diff --git a/apps/web/src/services/filmstrip/types.ts b/apps/web/src/services/filmstrip/types.ts
new file mode 100644
index 00000000..212caa73
--- /dev/null
+++ b/apps/web/src/services/filmstrip/types.ts
@@ -0,0 +1,25 @@
+export interface FilmstripFrame {
+ mediaId: string;
+ timestamp: number;
+ tier: number;
+ dataUrl: string;
+}
+
+export interface FilmstripRequest {
+ mediaId: string;
+ file: File;
+ timestamps: number[];
+}
+
+export type FilmstripStatus = "idle" | "generating" | "ready" | "error";
+
+export interface FilmstripGenerationProgress {
+ mediaId: string;
+ timestamp: number;
+ dataUrl: string;
+}
+
+export interface FilmstripGenerationComplete {
+ mediaId: string;
+ timestamps: number[];
+}
diff --git a/apps/web/src/services/filmstrip/worker.ts b/apps/web/src/services/filmstrip/worker.ts
new file mode 100644
index 00000000..e0d9ab9c
--- /dev/null
+++ b/apps/web/src/services/filmstrip/worker.ts
@@ -0,0 +1,141 @@
+import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
+import type { FilmstripRequest } from "./types";
+
+const THUMBNAIL_WIDTH = 106;
+const THUMBNAIL_HEIGHT = 60;
+const JPEG_QUALITY = 0.6;
+
+export type WorkerMessage =
+ | { type: "extract"; request: FilmstripRequest }
+ | { type: "cancel"; mediaId: string };
+
+export type WorkerResponse =
+ | {
+ type: "frame";
+ mediaId: string;
+ timestamp: number;
+ dataUrl: string;
+ }
+ | { type: "error"; mediaId: string; error: string }
+ | { type: "cancelled"; mediaId: string };
+
+const cancelledMediaIds = new Set();
+
+self.onmessage = async (event: MessageEvent) => {
+ const message = event.data;
+
+ switch (message.type) {
+ case "extract":
+ await handleExtract({ request: message.request });
+ break;
+ case "cancel":
+ cancelledMediaIds.add(message.mediaId);
+ self.postMessage({
+ type: "cancelled",
+ mediaId: message.mediaId,
+ } satisfies WorkerResponse);
+ break;
+ }
+};
+
+async function handleExtract({
+ request,
+}: {
+ request: FilmstripRequest;
+}): Promise {
+ const { mediaId, file, timestamps } = request;
+
+ if (cancelledMediaIds.has(mediaId)) {
+ cancelledMediaIds.delete(mediaId);
+ return;
+ }
+
+ try {
+ const input = new Input({
+ source: new BlobSource(file),
+ formats: ALL_FORMATS,
+ });
+
+ const videoTrack = await input.getPrimaryVideoTrack();
+ if (!videoTrack) {
+ self.postMessage({
+ type: "error",
+ mediaId,
+ error: "No video track found",
+ } satisfies WorkerResponse);
+ return;
+ }
+
+ const canDecode = await videoTrack.canDecode();
+ if (!canDecode) {
+ self.postMessage({
+ type: "error",
+ mediaId,
+ error: "Video codec not supported for decoding",
+ } satisfies WorkerResponse);
+ return;
+ }
+
+ const sink = new VideoSampleSink(videoTrack);
+
+ for (const timestamp of timestamps) {
+ if (cancelledMediaIds.has(mediaId)) {
+ cancelledMediaIds.delete(mediaId);
+ return;
+ }
+
+ const frame = await sink.getSample(timestamp);
+ if (!frame) {
+ continue;
+ }
+
+ try {
+ const dataUrl = await renderFrameToDataUrl({ frame });
+ self.postMessage({
+ type: "frame",
+ mediaId,
+ timestamp,
+ dataUrl,
+ } satisfies WorkerResponse);
+ } finally {
+ frame.close();
+ }
+ }
+ } catch (error) {
+ self.postMessage({
+ type: "error",
+ mediaId,
+ error:
+ error instanceof Error ? error.message : "Failed to extract frames",
+ } satisfies WorkerResponse);
+ }
+}
+
+async function renderFrameToDataUrl({
+ frame,
+}: {
+ frame: Awaited>;
+}): Promise {
+ const canvas = new OffscreenCanvas(THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT);
+ const context = canvas.getContext("2d");
+
+ if (!context) {
+ throw new Error("Could not get canvas context");
+ }
+
+ frame.draw(context, 0, 0, THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT);
+
+ const blob = await canvas.convertToBlob({
+ type: "image/jpeg",
+ quality: JPEG_QUALITY,
+ });
+
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onloadend = () => {
+ resolve(reader.result as string);
+ };
+ reader.onerror = reject;
+ reader.readAsDataURL(blob);
+ });
+}