395 lines
16 KiB
Svelte
395 lines
16 KiB
Svelte
<script lang="ts">
|
|
import { onMount } from 'svelte';
|
|
import { createEventDispatcher } from 'svelte';
|
|
import PhotoCrop from './PhotoCrop.svelte';
|
|
import * as tf from '@tensorflow/tfjs';
|
|
import * as blazeface from '@tensorflow-models/blazeface';
|
|
|
|
export let imageUrl: string;
|
|
export let personName: string;
|
|
export let isProcessing = false;
|
|
|
|
const dispatch = createEventDispatcher<{
|
|
cropUpdated: { x: number; y: number; width: number; height: number };
|
|
faceDetectionStarted: void;
|
|
faceDetectionCompleted: { success: boolean; hasAutoDetectedCrop: boolean };
|
|
}>();
|
|
|
|
let showCropEditor = false;
|
|
let autoDetectedCrop: { x: number; y: number; width: number; height: number } | null = null;
|
|
let currentCrop: { x: number; y: number; width: number; height: number } | null = null;
|
|
let isDetectingFace = false;
|
|
let faceDetectionError = false;
|
|
let detector: any = null;
|
|
let isModelLoading = false;
|
|
let isDownloadingModel = false;
|
|
|
|
let photoElement: HTMLImageElement;
|
|
|
|
onMount(async () => {
|
|
console.log('PhotoCard mounted, initializing face detection...');
|
|
await initializeFaceDetection();
|
|
});
|
|
|
|
async function initializeFaceDetection() {
|
|
try {
|
|
isDownloadingModel = true;
|
|
console.log('Downloading BlazeFace model...');
|
|
|
|
// Initialize TensorFlow.js with WebGL backend for better performance
|
|
await tf.setBackend('webgl');
|
|
await tf.ready();
|
|
console.log('TensorFlow.js WebGL backend initialized');
|
|
|
|
isDownloadingModel = false;
|
|
isModelLoading = true;
|
|
console.log('Loading BlazeFace model...');
|
|
|
|
// Load the BlazeFace model
|
|
detector = await blazeface.load();
|
|
|
|
isModelLoading = false;
|
|
console.log('BlazeFace model loaded successfully with WebGL backend');
|
|
} catch (error) {
|
|
console.error('Failed to initialize BlazeFace with WebGL:', error);
|
|
console.log('Falling back to CPU backend...');
|
|
|
|
try {
|
|
// Fallback to CPU if WebGL fails
|
|
await tf.setBackend('cpu');
|
|
await tf.ready();
|
|
console.log('TensorFlow.js CPU backend initialized as fallback');
|
|
|
|
detector = await blazeface.load();
|
|
isModelLoading = false;
|
|
console.log('BlazeFace model loaded successfully with CPU backend');
|
|
} catch (fallbackError) {
|
|
console.error('Failed to initialize BlazeFace with CPU fallback:', fallbackError);
|
|
isDownloadingModel = false;
|
|
isModelLoading = false;
|
|
faceDetectionError = true;
|
|
}
|
|
}
|
|
} // Simple face detection using BlazeFace
|
|
async function detectFaceWithMediaPipe() {
|
|
if (!photoElement || isDetectingFace || !detector) return;
|
|
|
|
dispatch('faceDetectionStarted');
|
|
isDetectingFace = true;
|
|
faceDetectionError = false;
|
|
|
|
try {
|
|
console.log('Detecting faces with BlazeFace...');
|
|
|
|
// Detect faces in the image
|
|
const predictions = await detector.estimateFaces(photoElement, false);
|
|
|
|
console.log(`BlazeFace found ${predictions.length} faces`);
|
|
|
|
if (predictions.length > 0) {
|
|
// Find the face with the highest probability
|
|
let bestFace = predictions[0];
|
|
let highestProbability = predictions[0].probability ? predictions[0].probability[0] : 0;
|
|
|
|
for (let i = 1; i < predictions.length; i++) {
|
|
const face = predictions[i];
|
|
const probability = face.probability ? face.probability[0] : 0;
|
|
if (probability > highestProbability) {
|
|
bestFace = face;
|
|
highestProbability = probability;
|
|
}
|
|
}
|
|
|
|
console.log(`Selected face with probability: ${highestProbability}`);
|
|
|
|
// Use the best detected face
|
|
const face = bestFace;
|
|
|
|
// BlazeFace returns topLeft and bottomRight coordinates
|
|
// These coordinates are relative to the DISPLAYED image size, not natural size
|
|
let [x1, y1] = face.topLeft;
|
|
let [x2, y2] = face.bottomRight;
|
|
|
|
console.log('BlazeFace detection (displayed coordinates):', { x1, y1, x2, y2 });
|
|
console.log('Image dimensions:', {
|
|
natural: { width: photoElement.naturalWidth, height: photoElement.naturalHeight },
|
|
displayed: { width: photoElement.clientWidth, height: photoElement.clientHeight }
|
|
});
|
|
|
|
// Calculate scale factors to convert from displayed to natural coordinates
|
|
const scaleX = photoElement.naturalWidth / photoElement.clientWidth;
|
|
const scaleY = photoElement.naturalHeight / photoElement.clientHeight;
|
|
|
|
console.log('Scale factors:', { scaleX, scaleY });
|
|
|
|
// Scale coordinates to natural image size
|
|
x1 = x1 * scaleX;
|
|
y1 = y1 * scaleY;
|
|
x2 = x2 * scaleX;
|
|
y2 = y2 * scaleY;
|
|
|
|
let faceWidth = x2 - x1;
|
|
let faceHeight = y2 - y1;
|
|
|
|
console.log('Scaled coordinates (natural size):', { x1, y1, x2, y2, faceWidth, faceHeight });
|
|
|
|
// BlazeFace coordinates are relative to the input image size
|
|
// Verify coordinates are within bounds and reasonable
|
|
if (x1 < 0 || y1 < 0 || x2 > photoElement.naturalWidth || y2 > photoElement.naturalHeight) {
|
|
console.warn('BlazeFace coordinates out of bounds, clamping:', { x1, y1, x2, y2 });
|
|
// Clamp coordinates to image bounds
|
|
x1 = Math.max(0, x1);
|
|
y1 = Math.max(0, y1);
|
|
x2 = Math.min(photoElement.naturalWidth, x2);
|
|
y2 = Math.min(photoElement.naturalHeight, y2);
|
|
|
|
// Recalculate dimensions
|
|
faceWidth = x2 - x1;
|
|
faceHeight = y2 - y1;
|
|
|
|
console.log('Clamped coordinates:', { x1, y1, x2, y2, faceWidth, faceHeight });
|
|
}
|
|
|
|
// Final validation - ensure we have a reasonable face size
|
|
if (faceWidth <= 0 || faceHeight <= 0) {
|
|
console.error('Invalid face dimensions after clamping');
|
|
throw new Error('Invalid face dimensions');
|
|
}
|
|
|
|
// Recalculate face dimensions after any clamping
|
|
const finalFaceWidth = x2 - x1;
|
|
const finalFaceHeight = y2 - y1;
|
|
|
|
// // Validate face size - reject if too small
|
|
// const faceArea = finalFaceWidth * finalFaceHeight;
|
|
// const imageArea = photoElement.naturalWidth * photoElement.naturalHeight;
|
|
// const faceRatio = faceArea / imageArea;
|
|
|
|
// console.log('Face area ratio:', faceRatio);
|
|
|
|
// // Only reject if smaller than 0.5% of image (very small noise)
|
|
// if (faceRatio < 0.005) {
|
|
// console.log('Face rejected: too small');
|
|
// throw new Error('Face too small');
|
|
// }
|
|
|
|
// Create crop area with environment-based configuration
|
|
const cropRatio = parseFloat(import.meta.env.VITE_CROP_RATIO || '1.0');
|
|
const faceOffsetX = parseFloat(import.meta.env.VITE_FACE_OFFSET_X || '0.0');
|
|
const faceOffsetY = parseFloat(import.meta.env.VITE_FACE_OFFSET_Y || '-0.1');
|
|
const cropScale = parseFloat(import.meta.env.VITE_CROP_SCALE || '2.5');
|
|
|
|
console.log('Crop configuration:', { cropRatio, faceOffsetX, faceOffsetY, cropScale });
|
|
|
|
// Calculate face center
|
|
const faceCenterX = x1 + finalFaceWidth / 2;
|
|
const faceCenterY = y1 + finalFaceHeight / 2;
|
|
|
|
// Calculate crop dimensions based on face width and scale
|
|
const cropWidth = finalFaceWidth * cropScale;
|
|
const cropHeight = cropWidth / cropRatio; // Maintain aspect ratio
|
|
|
|
// Apply face offset to crop center (offset is percentage of crop dimensions)
|
|
const cropCenterX = faceCenterX + (cropWidth * faceOffsetX);
|
|
const cropCenterY = faceCenterY + (cropHeight * faceOffsetY);
|
|
|
|
// Ensure crop fits within image bounds while maintaining aspect ratio
|
|
let finalCropWidth = cropWidth;
|
|
let finalCropHeight = cropHeight;
|
|
|
|
// Check if crop exceeds image bounds and scale down proportionally if needed
|
|
const maxWidth = photoElement.naturalWidth;
|
|
const maxHeight = photoElement.naturalHeight;
|
|
|
|
if (finalCropWidth > maxWidth || finalCropHeight > maxHeight) {
|
|
// Scale down to fit within bounds while maintaining ratio
|
|
const scaleToFitWidth = maxWidth / finalCropWidth;
|
|
const scaleToFitHeight = maxHeight / finalCropHeight;
|
|
const scaleToFit = Math.min(scaleToFitWidth, scaleToFitHeight);
|
|
|
|
finalCropWidth = finalCropWidth * scaleToFit;
|
|
finalCropHeight = finalCropHeight * scaleToFit;
|
|
|
|
console.log('Scaled crop to fit bounds:', { scaleToFit, finalCropWidth, finalCropHeight });
|
|
}
|
|
|
|
// Calculate crop position (top-left corner) with properly sized crop
|
|
const cropCenterXAdjusted = faceCenterX + (finalCropWidth * faceOffsetX);
|
|
const cropCenterYAdjusted = faceCenterY + (finalCropHeight * faceOffsetY);
|
|
|
|
const cropX = Math.max(0, Math.min(cropCenterXAdjusted - finalCropWidth / 2, photoElement.naturalWidth - finalCropWidth));
|
|
const cropY = Math.max(0, Math.min(cropCenterYAdjusted - finalCropHeight / 2, photoElement.naturalHeight - finalCropHeight));
|
|
|
|
console.log('Crop calculation:', {
|
|
faceCenter: { x: faceCenterX, y: faceCenterY },
|
|
cropDimensions: { width: cropWidth, height: cropHeight },
|
|
cropCenter: { x: cropCenterX, y: cropCenterY },
|
|
finalCrop: { x: cropX, y: cropY, width: finalCropWidth, height: finalCropHeight },
|
|
aspectRatio: finalCropWidth / finalCropHeight
|
|
});
|
|
|
|
autoDetectedCrop = {
|
|
x: Math.round(cropX),
|
|
y: Math.round(cropY),
|
|
width: Math.round(finalCropWidth),
|
|
height: Math.round(finalCropHeight)
|
|
};
|
|
|
|
currentCrop = { ...autoDetectedCrop };
|
|
dispatch('cropUpdated', currentCrop);
|
|
dispatch('faceDetectionCompleted', { success: true, hasAutoDetectedCrop: true });
|
|
console.log('BlazeFace detection successful!', autoDetectedCrop);
|
|
return;
|
|
}
|
|
|
|
// No faces detected
|
|
throw new Error('No faces detected by BlazeFace');
|
|
|
|
} catch (error) {
|
|
console.error('BlazeFace detection failed:', error);
|
|
faceDetectionError = true;
|
|
dispatch('faceDetectionCompleted', { success: false, hasAutoDetectedCrop: false });
|
|
// Don't fall back to anything - just leave it as an error state
|
|
} finally {
|
|
isDetectingFace = false;
|
|
}
|
|
}
|
|
function openCropEditor() {
|
|
showCropEditor = true;
|
|
}
|
|
|
|
function handleCropSave(e: CustomEvent<{ x: number; y: number; width: number; height: number }>) {
|
|
currentCrop = e.detail;
|
|
showCropEditor = false;
|
|
dispatch('cropUpdated', currentCrop);
|
|
}
|
|
|
|
function handleCropCancel() {
|
|
showCropEditor = false;
|
|
}
|
|
|
|
// Try face detection when image and detector are ready
|
|
$: if (imageUrl && photoElement && detector && !isDetectingFace && !autoDetectedCrop) {
|
|
detectFaceWithMediaPipe();
|
|
}
|
|
</script>
|
|
|
|
<div class="relative group">
|
|
<div class="relative overflow-hidden rounded-lg border-2 border-gray-200">
|
|
<img
|
|
bind:this={photoElement}
|
|
src={imageUrl}
|
|
alt={personName}
|
|
class="w-full h-full object-cover"
|
|
on:load={detectFaceWithMediaPipe}
|
|
/>
|
|
|
|
<!-- Small notification bars for all states -->
|
|
{#if isDownloadingModel}
|
|
<div class="absolute top-2 left-2 right-2 bg-blue-500/95 text-white px-3 py-2 rounded text-xs font-medium flex items-center space-x-2 shadow-lg">
|
|
<svg class="w-3 h-3 animate-spin" viewBox="0 0 24 24">
|
|
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4" fill="none"/>
|
|
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"/>
|
|
</svg>
|
|
<span>Downloading AI Model...</span>
|
|
<div class="flex-1 bg-white/20 rounded-full h-1 ml-2">
|
|
<div class="bg-white h-1 rounded-full animate-pulse" style="width: 30%"></div>
|
|
</div>
|
|
</div>
|
|
{:else if isModelLoading}
|
|
<div class="absolute top-2 left-2 right-2 bg-purple-500/95 text-white px-3 py-2 rounded text-xs font-medium flex items-center space-x-2 shadow-lg">
|
|
<svg class="w-3 h-3 animate-spin" viewBox="0 0 24 24">
|
|
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4" fill="none"/>
|
|
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"/>
|
|
</svg>
|
|
<span>Loading AI Model...</span>
|
|
<div class="flex-1 bg-white/20 rounded-full h-1 ml-2">
|
|
<div class="bg-white h-1 rounded-full animate-pulse" style="width: 60%"></div>
|
|
</div>
|
|
</div>
|
|
{:else if isDetectingFace}
|
|
<div class="absolute top-2 left-2 right-2 bg-green-500/95 text-white px-3 py-2 rounded text-xs font-medium flex items-center space-x-2 shadow-lg">
|
|
<svg class="w-3 h-3 animate-pulse" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
|
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M16 7a4 4 0 11-8 0 4 4 0 018 0zM12 14a7 7 0 00-7 7h14a7 7 0 00-7-7z"/>
|
|
</svg>
|
|
<span>Detecting Face
|
|
<span class="inline-flex ml-1">
|
|
<span class="animate-pulse">.</span>
|
|
<span class="animate-pulse" style="animation-delay: 0.2s">.</span>
|
|
<span class="animate-pulse" style="animation-delay: 0.4s">.</span>
|
|
</span>
|
|
</span>
|
|
<div class="flex-1 bg-white/20 rounded-full h-1 ml-2">
|
|
<div class="bg-white h-1 rounded-full animate-pulse" style="width: 80%"></div>
|
|
</div>
|
|
</div>
|
|
{/if}
|
|
|
|
{#if currentCrop}
|
|
<!-- Show crop preview overlay with proper masking -->
|
|
<div class="absolute inset-0 pointer-events-none">
|
|
<div class="relative w-full h-full">
|
|
<!-- Create mask using box-shadow to darken only non-crop areas -->
|
|
<div
|
|
class="absolute border-2 border-blue-500 border-dashed"
|
|
style="left: {(currentCrop.x / photoElement?.naturalWidth) * 100}%;
|
|
top: {(currentCrop.y / photoElement?.naturalHeight) * 100}%;
|
|
width: {(currentCrop.width / photoElement?.naturalWidth) * 100}%;
|
|
height: {(currentCrop.height / photoElement?.naturalHeight) * 100}%;
|
|
box-shadow: 0 0 0 9999px rgba(0, 0, 0, 0.3);"
|
|
></div>
|
|
</div>
|
|
</div>
|
|
{/if}
|
|
|
|
<!-- Edit crop button -->
|
|
<button
|
|
on:click={openCropEditor}
|
|
class="absolute top-2 right-2 bg-white bg-opacity-90 hover:bg-opacity-100 rounded-full p-2 shadow-lg transition-all duration-200 opacity-0 group-hover:opacity-100"
|
|
title="Edit crop area"
|
|
>
|
|
<svg class="w-4 h-4 text-gray-700" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
|
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15.232 5.232l3.536 3.536m-2.036-5.036a2.5 2.5 0 113.536 3.536L6.5 21.036H3v-3.572L16.732 3.732z"/>
|
|
</svg>
|
|
</button>
|
|
|
|
<!-- Status indicators -->
|
|
<div class="absolute bottom-2 left-2 flex space-x-1">
|
|
{#if faceDetectionError}
|
|
<div class="bg-yellow-500 text-white px-2 py-1 rounded text-xs font-medium">
|
|
Manual crop
|
|
</div>
|
|
{:else if currentCrop && autoDetectedCrop && JSON.stringify(currentCrop) !== JSON.stringify(autoDetectedCrop)}
|
|
<div class="bg-blue-500 text-white px-2 py-1 rounded text-xs font-medium">
|
|
Custom crop
|
|
</div>
|
|
{:else if autoDetectedCrop}
|
|
<div class="bg-green-500 text-white px-2 py-1 rounded text-xs font-medium">
|
|
Auto-cropped
|
|
</div>
|
|
{/if}
|
|
</div>
|
|
</div>
|
|
|
|
<div class="mt-2">
|
|
<p class="text-sm font-medium text-gray-900 truncate">{personName}</p>
|
|
{#if isProcessing}
|
|
<p class="text-xs text-gray-500">Processing...</p>
|
|
{:else if faceDetectionError}
|
|
<p class="text-xs text-yellow-600">Using center crop</p>
|
|
{:else if autoDetectedCrop}
|
|
<p class="text-xs text-green-600">Face detected</p>
|
|
{/if}
|
|
</div>
|
|
</div>
|
|
|
|
{#if showCropEditor}
|
|
<PhotoCrop
|
|
{imageUrl}
|
|
{personName}
|
|
initialCrop={currentCrop}
|
|
on:save={handleCropSave}
|
|
on:cancel={handleCropCancel}
|
|
/>
|
|
{/if}
|