From fc856e10c3b94f5c98d856671f260da56d92f829 Mon Sep 17 00:00:00 2001 From: Arnaud_Cayrol Date: Sat, 12 Apr 2025 13:33:42 +0200 Subject: [PATCH] Output a landmarks dict --- timelapse.py | 184 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 120 insertions(+), 64 deletions(-) diff --git a/timelapse.py b/timelapse.py index d10cc51..6688c8c 100644 --- a/timelapse.py +++ b/timelapse.py @@ -49,16 +49,16 @@ class ProcessConfig: min_face_width: int = 128 min_face_height: int = 128 pose_threshold: float = 25 - left_eye_pos: tuple = (0.4, 0.4) + left_eye_pos: tuple = (0.35, 0.4) landmark_model_path: str = "shape_predictor_68_face_landmarks.dat" -def draw_landmarks(image, shape, face_rect, output_path): +def draw_landmarks(image, landmarks, face_rect, output_path): """ Draws facial landmarks and face rectangle on the image and saves it. Args: image (numpy.ndarray): The input image in BGR format. - shape (dlib.full_object_detection): Detected facial landmarks. + landmarks (dict): Dictionary containing facial landmarks. face_rect (tuple): Face rectangle coordinates (x1, y1, x2, y2). output_path (str): Path to save the output image. """ @@ -70,11 +70,9 @@ def draw_landmarks(image, shape, face_rect, output_path): cv2.rectangle(img, (x1, y1), (x2, y2), (0, 255, 0), 2) # Draw landmarks if available - if shape is not None: - for i in range(68): - x = shape.part(i).x - y = shape.part(i).y - cv2.circle(img, (x, y), 2, (0, 0, 255), -1) + if landmarks is not None: + for i, (x, y) in landmarks.items(): + cv2.circle(img, (int(x), int(y)), 2, (0, 0, 255), -1) # Create output directory if it doesn't exist os.makedirs(os.path.dirname(output_path), exist_ok=True) @@ -198,12 +196,12 @@ def download_asset(api_key, base_url, asset_id): return response.content -def get_head_pose(shape, image): +def get_head_pose(landmarks, image): """ Estimates the head pose (pitch, yaw, roll) using facial landmarks. Args: - shape (dlib.full_object_detection): Detected facial landmarks. + landmarks (dict): Dictionary containing facial landmarks in numpy arrays. image (PIL.Image): The input image. Returns: @@ -213,12 +211,12 @@ def get_head_pose(shape, image): img_width, img_height = image.size image_points = np.array([ - (shape.part(30).x, shape.part(30).y), # Nose tip - (shape.part(8).x, shape.part(8).y), # Chin - (shape.part(36).x, shape.part(36).y), # Left eye left corner - (shape.part(45).x, shape.part(45).y), # Right eye right corner - (shape.part(48).x, shape.part(48).y), # Left Mouth corner - (shape.part(54).x, shape.part(54).y) # Right mouth corner + landmarks['nose_tip'], + landmarks['chin'], + landmarks['left_eye'][0], + landmarks['right_eye'][3], + landmarks['left_mouth'], + landmarks['right_mouth'] ], dtype="double") model_points = np.array([ @@ -261,8 +259,8 @@ def detect_landmarks(image, face_data, max_landmark_size=300): max_landmark_size (int): Maximum size for landmark detection. Returns: - dlib.full_object_detection or None: The detected facial landmarks in original image coordinates, - or None if detection failed. + dict or None: Dictionary containing facial landmarks in numpy arrays if successful, + None if detection failed. """ # Get face rectangle from metadata with proper scaling face_img_width = face_data.get("imageWidth") @@ -301,35 +299,50 @@ def detect_landmarks(image, face_data, max_landmark_size=300): if not shape: return None - # Scale landmarks back to original image coordinates - if scale_factor != 1.0: - for i in range(68): - shape.part(i).x = int(shape.part(i).x / scale_factor) - shape.part(i).y = int(shape.part(i).y / scale_factor) - - # Adjust coordinates to be relative to the original image + # Scale landmarks back to original image coordinates and adjust for face rectangle position + landmarks = {} for i in range(68): - shape.part(i).x += x1 - shape.part(i).y += y1 + x = shape.part(i).x + y = shape.part(i).y + if scale_factor != 1.0: + x = int(x / scale_factor) + y = int(y / scale_factor) + # Add the face rectangle's top-left coordinates to position landmarks correctly + x += x1 + y += y1 + landmarks[i] = (x, y) - return shape + # Convert to numpy arrays for specific facial features + left_eye = np.array([landmarks[i] for i in range(36, 42)]) + right_eye = np.array([landmarks[i] for i in range(42, 48)]) + nose_tip = np.array(landmarks[30]) + chin = np.array(landmarks[8]) + left_mouth = np.array(landmarks[48]) + right_mouth = np.array(landmarks[54]) + + return { + 'left_eye': left_eye, + 'right_eye': right_eye, + 'nose_tip': nose_tip, + 'chin': chin, + 'left_mouth': left_mouth, + 'right_mouth': right_mouth, + 'all_landmarks': landmarks + } -def check_eye_visibility(shape, ear_threshold=0.2): +def check_eye_visibility(left_eye, right_eye, ear_threshold=0.2): """ Checks if both eyes are visible by calculating the Eye Aspect Ratio (EAR). Args: - shape (dlib.full_object_detection): Detected facial landmarks. + left_eye (numpy.ndarray): Array of left eye landmarks. + right_eye (numpy.ndarray): Array of right eye landmarks. ear_threshold (float): Threshold for eye visibility. Returns: tuple: (left_ear, right_ear) if both eyes are visible, None otherwise. """ - # Get eye landmarks - left_eye = np.array([(shape.part(i).x, shape.part(i).y) for i in range(36, 42)]) - right_eye = np.array([(shape.part(i).x, shape.part(i).y) for i in range(42, 48)]) - def calculate_ear(eye): v1 = np.linalg.norm(eye[1] - eye[5]) v2 = np.linalg.norm(eye[2] - eye[4]) @@ -349,7 +362,7 @@ def check_eye_visibility(shape, ear_threshold=0.2): def crop_and_align_face(image, face_data, resize_size, face_resolution_threshold, pose_threshold, left_eye_pos): """ - Aligns a face in an image using facial landmarks and head pose estimation. + Aligns a face in an image by positioning the eyes at specified locations. Args: image (PIL.Image): The input image. @@ -357,61 +370,106 @@ def crop_and_align_face(image, face_data, resize_size, face_resolution_threshold resize_size (int): Size to resize the output image to. face_resolution_threshold (int): Minimum face resolution threshold. pose_threshold (float): Maximum allowed head pose deviation. - left_eye_pos (tuple): Desired position of the left eye in the output. + left_eye_pos (tuple): Desired position of the left eye in the output as percentages (x, y). Returns: PIL.Image or None: The aligned face image if successful, None otherwise. """ try: + # Get face rectangle from metadata with proper scaling + face_img_width = face_data.get("imageWidth") + face_img_height = face_data.get("imageHeight") + img_width, img_height = image.size + scale_x = img_width / face_img_width + scale_y = img_height / face_img_height + x1 = int(face_data.get("boundingBoxX1", 0) * scale_x) + x2 = int(face_data.get("boundingBoxX2", 0) * scale_x) + y1 = int(face_data.get("boundingBoxY1", 0) * scale_y) + y2 = int(face_data.get("boundingBoxY2", 0) * scale_y) + w = x2 - x1 + h = y2 - y1 + + # Convert image to numpy array for OpenCV processing + img_np = np.array(image) + img_np = cv2.cvtColor(img_np, cv2.COLOR_RGB2BGR) + # Detect landmarks in the face region - face_shape = detect_landmarks(image, face_data) - if not face_shape: + landmarks = detect_landmarks(image, face_data) + if not landmarks: logger.info("No facial landmarks detected") + draw_landmarks(img_np, None, (x1, y1, x2, y2), "discarded/no_landmarks.jpg") return None # Check if both eyes are visible - if not check_eye_visibility(face_shape): - logger.info("Eyes are too closed") + if not check_eye_visibility(landmarks['left_eye'], landmarks['right_eye']): + draw_landmarks(img_np, landmarks['all_landmarks'], (x1, y1, x2, y2), "discarded/side_face.jpg") return None # Get head pose - pose = get_head_pose(face_shape, image) + pose = get_head_pose(landmarks, image) if not pose: logger.info("Could not estimate head pose") - # draw_landmarks(np.array(image), shape, (x1, y1, x2, y2), "discarded/no_pose.jpg") + draw_landmarks(img_np, landmarks['all_landmarks'], (x1, y1, x2, y2), "discarded/no_pose.jpg") return None # Check if head pose is within acceptable range pitch, yaw, roll = pose if abs(pitch) > pose_threshold or abs(yaw) > pose_threshold or abs(roll) > pose_threshold: logger.info(f"Head pose exceeds threshold: pitch={pitch:.1f}°, yaw={yaw:.1f}°, roll={roll:.1f}°") - # draw_landmarks(np.array(image), shape, (x1, y1, x2, y2), f"discarded/pose_pitch_{pitch:.1f}_yaw_{yaw:.1f}_roll_{roll:.1f}.jpg") + draw_landmarks(img_np, landmarks['all_landmarks'], (x1, y1, x2, y2), f"discarded/pose_pitch_{pitch:.1f}_yaw_{yaw:.1f}_roll_{roll:.1f}.jpg") # return None - # Get eye positions for alignment - left_eye_center = np.mean([(face_shape.part(i).x, face_shape.part(i).y) for i in range(36, 42)], axis=0) - right_eye_center = np.mean([(face_shape.part(i).x, face_shape.part(i).y) for i in range(42, 48)], axis=0) + # Get eye positions + left_eye_center = np.array(np.mean(landmarks['left_eye'], axis=0)) + right_eye_center = np.array(np.mean(landmarks['right_eye'], axis=0)) - # Calculate angle between eyes - eye_angle = np.degrees(np.arctan2(right_eye_center[1] - left_eye_center[1], - right_eye_center[0] - left_eye_center[0])) + # Calculate the desired eye positions in the output image + left_eye_target = np.array([resize_size * left_eye_pos[0], resize_size * left_eye_pos[1]]) + right_eye_target = np.array([resize_size * (1.0 - left_eye_pos[0]), resize_size * left_eye_pos[1]]) - # Calculate rotation center (midpoint between eyes) - center = ((left_eye_center[0] + right_eye_center[0]) / 2, - (left_eye_center[1] + right_eye_center[1]) / 2) + # Calculate the angle between the current eye line and the target eye line + current_angle = np.degrees(np.arctan2(right_eye_center[1] - left_eye_center[1], + right_eye_center[0] - left_eye_center[0])) + target_angle = np.degrees(np.arctan2(right_eye_target[1] - left_eye_target[1], + right_eye_target[0] - left_eye_target[0])) + rotation_angle = target_angle - current_angle - # Create rotation matrix - rotation_matrix = cv2.getRotationMatrix2D(center, eye_angle, 1.0) + # Calculate the scale factor to match the desired eye distance + current_eye_distance = np.linalg.norm(right_eye_center - left_eye_center) + target_eye_distance = np.linalg.norm(right_eye_target - left_eye_target) + scale = target_eye_distance / current_eye_distance - # Calculate translation to position left eye - tx = resize_size * left_eye_pos[0] - left_eye_center[0] - ty = resize_size * left_eye_pos[1] - left_eye_center[1] - rotation_matrix[0, 2] += tx - rotation_matrix[1, 2] += ty + # Create the transformation matrix + # First, translate to origin (center of eyes) + center = np.array([(left_eye_center[0] + right_eye_center[0]) / 2, + (left_eye_center[1] + right_eye_center[1]) / 2]) + M1 = np.array([[1, 0, -center[0]], + [0, 1, -center[1]], + [0, 0, 1]]) - # Convert image to numpy array for OpenCV processing - img_np = np.array(image) - img_np = cv2.cvtColor(img_np, cv2.COLOR_RGB2BGR) + # Then rotate + angle_rad = np.radians(rotation_angle) + M2 = np.array([[np.cos(angle_rad), -np.sin(angle_rad), 0], + [np.sin(angle_rad), np.cos(angle_rad), 0], + [0, 0, 1]]) + + # Then scale + M3 = np.array([[scale, 0, 0], + [0, scale, 0], + [0, 0, 1]]) + + # Finally, translate to target position + target_center = np.array([(left_eye_target[0] + right_eye_target[0]) / 2, + (left_eye_target[1] + right_eye_target[1]) / 2]) + M4 = np.array([[1, 0, target_center[0]], + [0, 1, target_center[1]], + [0, 0, 1]]) + + # Combine all transformations + M = M4 @ M3 @ M2 @ M1 + + # Convert to 2x3 matrix for OpenCV + rotation_matrix = M[:2, :] # Apply transformation aligned_face = cv2.warpAffine(img_np, rotation_matrix, (resize_size, resize_size), @@ -421,8 +479,6 @@ def crop_and_align_face(image, face_data, resize_size, face_resolution_threshold aligned_face = cv2.cvtColor(aligned_face, cv2.COLOR_BGR2RGB) aligned_face = Image.fromarray(aligned_face) - # draw_landmarks(img_np, shape, (x1, y1, x2, y2), "final/aligned_face.jpg") - return aligned_face except Exception as e: