123 lines
3.2 KiB
TypeScript
123 lines
3.2 KiB
TypeScript
import type { EffectContext } from "@opencut/effects";
|
|
|
|
/**
|
|
* Face mesh detection provider using MediaPipe Face Mesh.
|
|
* Lazy-loads the WASM module only when first needed.
|
|
* Runs detection per frame and caches results.
|
|
*/
|
|
|
|
import type { FaceMesh as FaceMeshType, Results } from "@mediapipe/face_mesh";
|
|
|
|
let faceMeshInstance: FaceMeshType | null = null;
|
|
let isLoading = false;
|
|
/** Resolve function for the current pending detection — avoids race conditions */
|
|
let pendingResolve: ((results: Results) => void) | null = null;
|
|
|
|
/** Lazy-load MediaPipe Face Mesh WASM module */
|
|
async function loadFaceMesh(): Promise<FaceMeshType | null> {
|
|
if (faceMeshInstance) return faceMeshInstance;
|
|
if (isLoading) return null;
|
|
|
|
isLoading = true;
|
|
try {
|
|
const { FaceMesh } = await import("@mediapipe/face_mesh");
|
|
const fm = new FaceMesh({
|
|
locateFile: (file: string) =>
|
|
`https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/${file}`,
|
|
});
|
|
fm.setOptions({
|
|
maxNumFaces: 1,
|
|
refineLandmarks: true,
|
|
minDetectionConfidence: 0.5,
|
|
minTrackingConfidence: 0.5,
|
|
});
|
|
fm.onResults((results: Results) => {
|
|
if (pendingResolve) {
|
|
pendingResolve(results);
|
|
pendingResolve = null;
|
|
}
|
|
});
|
|
faceMeshInstance = fm;
|
|
return fm;
|
|
} catch (err) {
|
|
console.warn("[face-mesh] Failed to load MediaPipe:", err);
|
|
return null;
|
|
} finally {
|
|
isLoading = false;
|
|
}
|
|
}
|
|
|
|
/** MediaPipe face landmark indices for key regions */
|
|
const LANDMARK_INDICES = {
|
|
leftCheek: 234,
|
|
rightCheek: 454,
|
|
jawBottom: 152,
|
|
jawLeft: 132,
|
|
jawRight: 361,
|
|
leftEyeCenter: 159,
|
|
rightEyeCenter: 386,
|
|
mouthCenter: 13,
|
|
};
|
|
|
|
/** Convert MediaPipe face landmarks to EffectContext */
|
|
function landmarksToContext(
|
|
landmarks: Array<{ x: number; y: number; z: number }>,
|
|
): EffectContext {
|
|
const lc = landmarks[LANDMARK_INDICES.leftCheek];
|
|
const rc = landmarks[LANDMARK_INDICES.rightCheek];
|
|
const jaw = landmarks[LANDMARK_INDICES.jawBottom];
|
|
const jawL = landmarks[LANDMARK_INDICES.jawLeft];
|
|
const jawR = landmarks[LANDMARK_INDICES.jawRight];
|
|
|
|
// Estimate cheek radius from face width
|
|
const faceWidth = Math.abs(rc.x - lc.x);
|
|
const cheekRadius = faceWidth * 0.15;
|
|
|
|
return {
|
|
faceDetected: true,
|
|
cheekLeft: [lc.x, lc.y],
|
|
cheekRight: [rc.x, rc.y],
|
|
cheekRadius,
|
|
jawPoints: [jaw.x, jaw.y, jawL.x, jawL.y, jawR.x, jawR.y],
|
|
};
|
|
}
|
|
|
|
/** Detect face in the given image source and return EffectContext */
|
|
export async function detectFace(
|
|
source: CanvasImageSource,
|
|
): Promise<EffectContext> {
|
|
const fm = await loadFaceMesh();
|
|
if (!fm) {
|
|
return { faceDetected: false };
|
|
}
|
|
|
|
// Promise-based approach avoids race conditions with concurrent calls
|
|
const results = await new Promise<Results>((resolve) => {
|
|
pendingResolve = resolve;
|
|
fm.send({ image: source as HTMLCanvasElement });
|
|
});
|
|
|
|
if (
|
|
!results?.multiFaceLandmarks ||
|
|
results.multiFaceLandmarks.length === 0
|
|
) {
|
|
return { faceDetected: false };
|
|
}
|
|
|
|
return landmarksToContext(results.multiFaceLandmarks[0]);
|
|
}
|
|
|
|
/** Check if MediaPipe is loaded (for conditional rendering) */
|
|
export function isFaceMeshReady(): boolean {
|
|
return faceMeshInstance !== null;
|
|
}
|
|
|
|
/** Clean up MediaPipe resources */
|
|
export function disposeFaceMesh(): void {
|
|
if (faceMeshInstance) {
|
|
faceMeshInstance.close();
|
|
faceMeshInstance = null;
|
|
}
|
|
pendingResolve = null;
|
|
}
|