307 lines
8.8 KiB
TypeScript
307 lines
8.8 KiB
TypeScript
/**
|
|
* Hand Posture Detection Service
|
|
* Uses ml5.js handpose model to detect hand posture and provide feedback
|
|
*/
|
|
|
|
export interface HandLandmarks {
|
|
keypoints: number[][]; // Array of [x, y, z] coordinates for each hand landmark
|
|
handedness: 'Left' | 'Right';
|
|
confidence: number;
|
|
}
|
|
|
|
export interface PostureAnalysis {
|
|
isCorrect: boolean;
|
|
score: number; // 0-100
|
|
issues: PostureIssue[];
|
|
recommendations: string[];
|
|
}
|
|
|
|
export interface PostureIssue {
|
|
type: 'wrist' | 'fingers' | 'palm' | 'thumb';
|
|
severity: 'low' | 'medium' | 'high';
|
|
message: string;
|
|
suggestedFix: string;
|
|
}
|
|
|
|
/**
|
|
* Analyze hand posture for piano playing
|
|
* Based on classical piano technique principles
|
|
*/
|
|
export class HandPostureAnalyzer {
|
|
/**
|
|
* Analyze complete hand posture
|
|
*/
|
|
static analyzePosture(landmarks: HandLandmarks): PostureAnalysis {
|
|
const issues: PostureIssue[] = [];
|
|
let score = 100;
|
|
|
|
// Analyze wrist position
|
|
const wristAnalysis = this.analyzeWrist(landmarks.keypoints);
|
|
if (!wristAnalysis.isCorrect) {
|
|
issues.push(wristAnalysis.issue!);
|
|
score -= wristAnalysis.issue!.severity === 'high' ? 30 : 15;
|
|
}
|
|
|
|
// Analyze finger curvature
|
|
const fingerAnalysis = this.analyzeFingerCurvature(landmarks.keypoints);
|
|
if (!fingerAnalysis.isCorrect) {
|
|
issues.push(fingerAnalysis.issue!);
|
|
score -= fingerAnalysis.issue!.severity === 'high' ? 25 : 10;
|
|
}
|
|
|
|
// Analyze palm position
|
|
const palmAnalysis = this.analyzePalmPosition(landmarks.keypoints);
|
|
if (!palmAnalysis.isCorrect) {
|
|
issues.push(palmAnalysis.issue!);
|
|
score -= palmAnalysis.issue!.severity === 'high' ? 20 : 10;
|
|
}
|
|
|
|
// Analyze thumb position
|
|
const thumbAnalysis = this.analyzeThumbPosition(landmarks.keypoints);
|
|
if (!thumbAnalysis.isCorrect) {
|
|
issues.push(thumbAnalysis.issue!);
|
|
score -= thumbAnalysis.issue!.severity === 'high' ? 15 : 5;
|
|
}
|
|
|
|
// Generate recommendations
|
|
const recommendations = this.generateRecommendations(issues);
|
|
|
|
return {
|
|
isCorrect: issues.length === 0,
|
|
score: Math.max(0, score),
|
|
issues,
|
|
recommendations,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Analyze wrist position (should be level with hand, not too low)
|
|
*/
|
|
private static analyzeWrist(keypoints: number[][]): { isCorrect: boolean; issue?: PostureIssue } {
|
|
const wrist = keypoints[0]; // Wrist keypoint
|
|
const middleFingerBase = keypoints[9]; // Middle finger MCP joint
|
|
|
|
// Calculate vertical distance (Y-axis)
|
|
const verticalDiff = middleFingerBase[1] - wrist[1];
|
|
|
|
// Wrist should be roughly level or slightly above finger base
|
|
if (verticalDiff < -30) {
|
|
return {
|
|
isCorrect: false,
|
|
issue: {
|
|
type: 'wrist',
|
|
severity: 'high',
|
|
message: 'Pulso muito baixo',
|
|
suggestedFix:
|
|
'Levante o pulso até ficar no mesmo nível dos dedos. Imagine uma linha reta do cotovelo até os dedos.',
|
|
},
|
|
};
|
|
}
|
|
|
|
if (verticalDiff > 40) {
|
|
return {
|
|
isCorrect: false,
|
|
issue: {
|
|
type: 'wrist',
|
|
severity: 'medium',
|
|
message: 'Pulso muito alto',
|
|
suggestedFix: 'Abaixe levemente o pulso. O pulso deve estar relaxado, não tenso.',
|
|
},
|
|
};
|
|
}
|
|
|
|
return { isCorrect: true };
|
|
}
|
|
|
|
/**
|
|
* Analyze finger curvature (should be naturally curved, not flat)
|
|
*/
|
|
private static analyzeFingerCurvature(keypoints: number[][]): {
|
|
isCorrect: boolean;
|
|
issue?: PostureIssue;
|
|
} {
|
|
// Analyze index finger as representative
|
|
const indexBase = keypoints[5]; // Index MCP
|
|
const indexMiddle = keypoints[6]; // Index PIP
|
|
const indexTip = keypoints[8]; // Index DIP
|
|
|
|
// Calculate curvature using the middle joint position
|
|
const baseToTipDistance = Math.hypot(indexTip[0] - indexBase[0], indexTip[1] - indexBase[1]);
|
|
|
|
const baseToMiddleDistance = Math.hypot(
|
|
indexMiddle[0] - indexBase[0],
|
|
indexMiddle[1] - indexBase[1]
|
|
);
|
|
|
|
const curvatureRatio = baseToMiddleDistance / baseToTipDistance;
|
|
|
|
// Too flat (ratio close to 1)
|
|
if (curvatureRatio > 0.8) {
|
|
return {
|
|
isCorrect: false,
|
|
issue: {
|
|
type: 'fingers',
|
|
severity: 'high',
|
|
message: 'Dedos muito retos',
|
|
suggestedFix:
|
|
'Curve os dedos naturalmente, como se estivesse segurando uma bola. Isso dá mais controle e previne lesões.',
|
|
},
|
|
};
|
|
}
|
|
|
|
// Too curved (ratio too small)
|
|
if (curvatureRatio < 0.4) {
|
|
return {
|
|
isCorrect: false,
|
|
issue: {
|
|
type: 'fingers',
|
|
severity: 'medium',
|
|
message: 'Dedos muito curvados',
|
|
suggestedFix: 'Relaxe os dedos um pouco. A curvatura deve ser natural, não forçada.',
|
|
},
|
|
};
|
|
}
|
|
|
|
return { isCorrect: true };
|
|
}
|
|
|
|
/**
|
|
* Analyze palm position (should be arched, not collapsed)
|
|
*/
|
|
private static analyzePalmPosition(keypoints: number[][]): {
|
|
isCorrect: boolean;
|
|
issue?: PostureIssue;
|
|
} {
|
|
const wrist = keypoints[0];
|
|
const indexBase = keypoints[5];
|
|
const pinkyBase = keypoints[17];
|
|
|
|
// Calculate palm width
|
|
const palmWidth = Math.hypot(pinkyBase[0] - indexBase[0], pinkyBase[1] - indexBase[1]);
|
|
|
|
// Calculate palm center to wrist distance
|
|
const palmCenterX = (indexBase[0] + pinkyBase[0]) / 2;
|
|
const palmCenterY = (indexBase[1] + pinkyBase[1]) / 2;
|
|
const palmHeight = Math.hypot(palmCenterX - wrist[0], palmCenterY - wrist[1]);
|
|
|
|
// Palm should have an arch (height/width ratio)
|
|
const archRatio = palmHeight / palmWidth;
|
|
|
|
if (archRatio < 0.6) {
|
|
return {
|
|
isCorrect: false,
|
|
issue: {
|
|
type: 'palm',
|
|
severity: 'medium',
|
|
message: 'Palma da mão muito achatada',
|
|
suggestedFix: 'Crie um arco natural na palma da mão. Imagine segurar uma laranja.',
|
|
},
|
|
};
|
|
}
|
|
|
|
return { isCorrect: true };
|
|
}
|
|
|
|
/**
|
|
* Analyze thumb position (should be relaxed and rounded, not tucked under)
|
|
*/
|
|
private static analyzeThumbPosition(keypoints: number[][]): {
|
|
isCorrect: boolean;
|
|
issue?: PostureIssue;
|
|
} {
|
|
const thumbBase = keypoints[1];
|
|
const thumbTip = keypoints[4];
|
|
const indexBase = keypoints[5];
|
|
|
|
// Calculate thumb extension
|
|
const thumbToIndex = Math.hypot(thumbTip[0] - indexBase[0], thumbTip[1] - indexBase[1]);
|
|
|
|
const thumbLength = Math.hypot(thumbTip[0] - thumbBase[0], thumbTip[1] - thumbBase[1]);
|
|
|
|
const extensionRatio = thumbToIndex / thumbLength;
|
|
|
|
// Thumb too close to fingers (tucked)
|
|
if (extensionRatio < 0.7) {
|
|
return {
|
|
isCorrect: false,
|
|
issue: {
|
|
type: 'thumb',
|
|
severity: 'medium',
|
|
message: 'Polegar muito próximo dos outros dedos',
|
|
suggestedFix:
|
|
'Mantenha o polegar mais afastado e relaxado, formando um espaço natural com o indicador.',
|
|
},
|
|
};
|
|
}
|
|
|
|
// Thumb too extended
|
|
if (extensionRatio > 1.8) {
|
|
return {
|
|
isCorrect: false,
|
|
issue: {
|
|
type: 'thumb',
|
|
severity: 'low',
|
|
message: 'Polegar muito estendido',
|
|
suggestedFix: 'Relaxe o polegar, mantendo-o em uma posição mais natural.',
|
|
},
|
|
};
|
|
}
|
|
|
|
return { isCorrect: true };
|
|
}
|
|
|
|
/**
|
|
* Generate actionable recommendations based on issues
|
|
*/
|
|
private static generateRecommendations(issues: PostureIssue[]): string[] {
|
|
const recommendations: string[] = [];
|
|
|
|
if (issues.length === 0) {
|
|
return ['Excelente postura! Continue assim. 👏'];
|
|
}
|
|
|
|
// Sort by severity
|
|
const sortedIssues = issues.sort((a, b) => {
|
|
const severityOrder = { high: 0, medium: 1, low: 2 };
|
|
return severityOrder[a.severity] - severityOrder[b.severity];
|
|
});
|
|
|
|
// Add primary recommendation (most severe issue)
|
|
recommendations.push(`🎯 Prioridade: ${sortedIssues[0].suggestedFix}`);
|
|
|
|
// Add secondary recommendations
|
|
if (sortedIssues.length > 1) {
|
|
recommendations.push(`➡️ Depois: ${sortedIssues[1].suggestedFix}`);
|
|
}
|
|
|
|
// Add general tip
|
|
if (issues.some(i => i.severity === 'high')) {
|
|
recommendations.push(
|
|
'💡 Dica: Pratique a postura na frente de um espelho por 5 minutos antes de tocar.'
|
|
);
|
|
}
|
|
|
|
return recommendations;
|
|
}
|
|
|
|
/**
|
|
* Get visual feedback color based on score
|
|
*/
|
|
static getScoreColor(score: number): string {
|
|
if (score >= 80) return '#10b981'; // Green - Good
|
|
if (score >= 60) return '#f59e0b'; // Yellow - Needs improvement
|
|
return '#ef4444'; // Red - Poor
|
|
}
|
|
|
|
/**
|
|
* Get visual feedback message based on score
|
|
*/
|
|
static getScoreMessage(score: number): string {
|
|
if (score >= 90) return 'Postura Excelente! 🎯';
|
|
if (score >= 80) return 'Boa Postura! ✅';
|
|
if (score >= 70) return 'Postura Aceitável 👍';
|
|
if (score >= 60) return 'Precisa Melhorar ⚠️';
|
|
return 'Atenção à Postura! ❌';
|
|
}
|
|
}
|