Output a landmarks dict
This commit is contained in:
Arnaud_Cayrol
parent
e72070b75e
commit
fc856e10c3
1 changed files with 120 additions and 64 deletions
184
timelapse.py
184
timelapse.py
|
|
@ -49,16 +49,16 @@ class ProcessConfig:
|
|||
min_face_width: int = 128
|
||||
min_face_height: int = 128
|
||||
pose_threshold: float = 25
|
||||
left_eye_pos: tuple = (0.4, 0.4)
|
||||
left_eye_pos: tuple = (0.35, 0.4)
|
||||
landmark_model_path: str = "shape_predictor_68_face_landmarks.dat"
|
||||
|
||||
def draw_landmarks(image, shape, face_rect, output_path):
|
||||
def draw_landmarks(image, landmarks, face_rect, output_path):
|
||||
"""
|
||||
Draws facial landmarks and face rectangle on the image and saves it.
|
||||
|
||||
Args:
|
||||
image (numpy.ndarray): The input image in BGR format.
|
||||
shape (dlib.full_object_detection): Detected facial landmarks.
|
||||
landmarks (dict): Dictionary containing facial landmarks.
|
||||
face_rect (tuple): Face rectangle coordinates (x1, y1, x2, y2).
|
||||
output_path (str): Path to save the output image.
|
||||
"""
|
||||
|
|
@ -70,11 +70,9 @@ def draw_landmarks(image, shape, face_rect, output_path):
|
|||
cv2.rectangle(img, (x1, y1), (x2, y2), (0, 255, 0), 2)
|
||||
|
||||
# Draw landmarks if available
|
||||
if shape is not None:
|
||||
for i in range(68):
|
||||
x = shape.part(i).x
|
||||
y = shape.part(i).y
|
||||
cv2.circle(img, (x, y), 2, (0, 0, 255), -1)
|
||||
if landmarks is not None:
|
||||
for i, (x, y) in landmarks.items():
|
||||
cv2.circle(img, (int(x), int(y)), 2, (0, 0, 255), -1)
|
||||
|
||||
# Create output directory if it doesn't exist
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
|
|
@ -198,12 +196,12 @@ def download_asset(api_key, base_url, asset_id):
|
|||
return response.content
|
||||
|
||||
|
||||
def get_head_pose(shape, image):
|
||||
def get_head_pose(landmarks, image):
|
||||
"""
|
||||
Estimates the head pose (pitch, yaw, roll) using facial landmarks.
|
||||
|
||||
Args:
|
||||
shape (dlib.full_object_detection): Detected facial landmarks.
|
||||
landmarks (dict): Dictionary containing facial landmarks in numpy arrays.
|
||||
image (PIL.Image): The input image.
|
||||
|
||||
Returns:
|
||||
|
|
@ -213,12 +211,12 @@ def get_head_pose(shape, image):
|
|||
img_width, img_height = image.size
|
||||
|
||||
image_points = np.array([
|
||||
(shape.part(30).x, shape.part(30).y), # Nose tip
|
||||
(shape.part(8).x, shape.part(8).y), # Chin
|
||||
(shape.part(36).x, shape.part(36).y), # Left eye left corner
|
||||
(shape.part(45).x, shape.part(45).y), # Right eye right corner
|
||||
(shape.part(48).x, shape.part(48).y), # Left Mouth corner
|
||||
(shape.part(54).x, shape.part(54).y) # Right mouth corner
|
||||
landmarks['nose_tip'],
|
||||
landmarks['chin'],
|
||||
landmarks['left_eye'][0],
|
||||
landmarks['right_eye'][3],
|
||||
landmarks['left_mouth'],
|
||||
landmarks['right_mouth']
|
||||
], dtype="double")
|
||||
|
||||
model_points = np.array([
|
||||
|
|
@ -261,8 +259,8 @@ def detect_landmarks(image, face_data, max_landmark_size=300):
|
|||
max_landmark_size (int): Maximum size for landmark detection.
|
||||
|
||||
Returns:
|
||||
dlib.full_object_detection or None: The detected facial landmarks in original image coordinates,
|
||||
or None if detection failed.
|
||||
dict or None: Dictionary containing facial landmarks in numpy arrays if successful,
|
||||
None if detection failed.
|
||||
"""
|
||||
# Get face rectangle from metadata with proper scaling
|
||||
face_img_width = face_data.get("imageWidth")
|
||||
|
|
@ -301,35 +299,50 @@ def detect_landmarks(image, face_data, max_landmark_size=300):
|
|||
if not shape:
|
||||
return None
|
||||
|
||||
# Scale landmarks back to original image coordinates
|
||||
if scale_factor != 1.0:
|
||||
for i in range(68):
|
||||
shape.part(i).x = int(shape.part(i).x / scale_factor)
|
||||
shape.part(i).y = int(shape.part(i).y / scale_factor)
|
||||
|
||||
# Adjust coordinates to be relative to the original image
|
||||
# Scale landmarks back to original image coordinates and adjust for face rectangle position
|
||||
landmarks = {}
|
||||
for i in range(68):
|
||||
shape.part(i).x += x1
|
||||
shape.part(i).y += y1
|
||||
x = shape.part(i).x
|
||||
y = shape.part(i).y
|
||||
if scale_factor != 1.0:
|
||||
x = int(x / scale_factor)
|
||||
y = int(y / scale_factor)
|
||||
# Add the face rectangle's top-left coordinates to position landmarks correctly
|
||||
x += x1
|
||||
y += y1
|
||||
landmarks[i] = (x, y)
|
||||
|
||||
return shape
|
||||
# Convert to numpy arrays for specific facial features
|
||||
left_eye = np.array([landmarks[i] for i in range(36, 42)])
|
||||
right_eye = np.array([landmarks[i] for i in range(42, 48)])
|
||||
nose_tip = np.array(landmarks[30])
|
||||
chin = np.array(landmarks[8])
|
||||
left_mouth = np.array(landmarks[48])
|
||||
right_mouth = np.array(landmarks[54])
|
||||
|
||||
return {
|
||||
'left_eye': left_eye,
|
||||
'right_eye': right_eye,
|
||||
'nose_tip': nose_tip,
|
||||
'chin': chin,
|
||||
'left_mouth': left_mouth,
|
||||
'right_mouth': right_mouth,
|
||||
'all_landmarks': landmarks
|
||||
}
|
||||
|
||||
|
||||
def check_eye_visibility(shape, ear_threshold=0.2):
|
||||
def check_eye_visibility(left_eye, right_eye, ear_threshold=0.2):
|
||||
"""
|
||||
Checks if both eyes are visible by calculating the Eye Aspect Ratio (EAR).
|
||||
|
||||
Args:
|
||||
shape (dlib.full_object_detection): Detected facial landmarks.
|
||||
left_eye (numpy.ndarray): Array of left eye landmarks.
|
||||
right_eye (numpy.ndarray): Array of right eye landmarks.
|
||||
ear_threshold (float): Threshold for eye visibility.
|
||||
|
||||
Returns:
|
||||
tuple: (left_ear, right_ear) if both eyes are visible, None otherwise.
|
||||
"""
|
||||
# Get eye landmarks
|
||||
left_eye = np.array([(shape.part(i).x, shape.part(i).y) for i in range(36, 42)])
|
||||
right_eye = np.array([(shape.part(i).x, shape.part(i).y) for i in range(42, 48)])
|
||||
|
||||
def calculate_ear(eye):
|
||||
v1 = np.linalg.norm(eye[1] - eye[5])
|
||||
v2 = np.linalg.norm(eye[2] - eye[4])
|
||||
|
|
@ -349,7 +362,7 @@ def check_eye_visibility(shape, ear_threshold=0.2):
|
|||
|
||||
def crop_and_align_face(image, face_data, resize_size, face_resolution_threshold, pose_threshold, left_eye_pos):
|
||||
"""
|
||||
Aligns a face in an image using facial landmarks and head pose estimation.
|
||||
Aligns a face in an image by positioning the eyes at specified locations.
|
||||
|
||||
Args:
|
||||
image (PIL.Image): The input image.
|
||||
|
|
@ -357,61 +370,106 @@ def crop_and_align_face(image, face_data, resize_size, face_resolution_threshold
|
|||
resize_size (int): Size to resize the output image to.
|
||||
face_resolution_threshold (int): Minimum face resolution threshold.
|
||||
pose_threshold (float): Maximum allowed head pose deviation.
|
||||
left_eye_pos (tuple): Desired position of the left eye in the output.
|
||||
left_eye_pos (tuple): Desired position of the left eye in the output as percentages (x, y).
|
||||
|
||||
Returns:
|
||||
PIL.Image or None: The aligned face image if successful, None otherwise.
|
||||
"""
|
||||
try:
|
||||
# Get face rectangle from metadata with proper scaling
|
||||
face_img_width = face_data.get("imageWidth")
|
||||
face_img_height = face_data.get("imageHeight")
|
||||
img_width, img_height = image.size
|
||||
scale_x = img_width / face_img_width
|
||||
scale_y = img_height / face_img_height
|
||||
x1 = int(face_data.get("boundingBoxX1", 0) * scale_x)
|
||||
x2 = int(face_data.get("boundingBoxX2", 0) * scale_x)
|
||||
y1 = int(face_data.get("boundingBoxY1", 0) * scale_y)
|
||||
y2 = int(face_data.get("boundingBoxY2", 0) * scale_y)
|
||||
w = x2 - x1
|
||||
h = y2 - y1
|
||||
|
||||
# Convert image to numpy array for OpenCV processing
|
||||
img_np = np.array(image)
|
||||
img_np = cv2.cvtColor(img_np, cv2.COLOR_RGB2BGR)
|
||||
|
||||
# Detect landmarks in the face region
|
||||
face_shape = detect_landmarks(image, face_data)
|
||||
if not face_shape:
|
||||
landmarks = detect_landmarks(image, face_data)
|
||||
if not landmarks:
|
||||
logger.info("No facial landmarks detected")
|
||||
draw_landmarks(img_np, None, (x1, y1, x2, y2), "discarded/no_landmarks.jpg")
|
||||
return None
|
||||
|
||||
# Check if both eyes are visible
|
||||
if not check_eye_visibility(face_shape):
|
||||
logger.info("Eyes are too closed")
|
||||
if not check_eye_visibility(landmarks['left_eye'], landmarks['right_eye']):
|
||||
draw_landmarks(img_np, landmarks['all_landmarks'], (x1, y1, x2, y2), "discarded/side_face.jpg")
|
||||
return None
|
||||
|
||||
# Get head pose
|
||||
pose = get_head_pose(face_shape, image)
|
||||
pose = get_head_pose(landmarks, image)
|
||||
if not pose:
|
||||
logger.info("Could not estimate head pose")
|
||||
# draw_landmarks(np.array(image), shape, (x1, y1, x2, y2), "discarded/no_pose.jpg")
|
||||
draw_landmarks(img_np, landmarks['all_landmarks'], (x1, y1, x2, y2), "discarded/no_pose.jpg")
|
||||
return None
|
||||
|
||||
# Check if head pose is within acceptable range
|
||||
pitch, yaw, roll = pose
|
||||
if abs(pitch) > pose_threshold or abs(yaw) > pose_threshold or abs(roll) > pose_threshold:
|
||||
logger.info(f"Head pose exceeds threshold: pitch={pitch:.1f}°, yaw={yaw:.1f}°, roll={roll:.1f}°")
|
||||
# draw_landmarks(np.array(image), shape, (x1, y1, x2, y2), f"discarded/pose_pitch_{pitch:.1f}_yaw_{yaw:.1f}_roll_{roll:.1f}.jpg")
|
||||
draw_landmarks(img_np, landmarks['all_landmarks'], (x1, y1, x2, y2), f"discarded/pose_pitch_{pitch:.1f}_yaw_{yaw:.1f}_roll_{roll:.1f}.jpg")
|
||||
# return None
|
||||
|
||||
# Get eye positions for alignment
|
||||
left_eye_center = np.mean([(face_shape.part(i).x, face_shape.part(i).y) for i in range(36, 42)], axis=0)
|
||||
right_eye_center = np.mean([(face_shape.part(i).x, face_shape.part(i).y) for i in range(42, 48)], axis=0)
|
||||
# Get eye positions
|
||||
left_eye_center = np.array(np.mean(landmarks['left_eye'], axis=0))
|
||||
right_eye_center = np.array(np.mean(landmarks['right_eye'], axis=0))
|
||||
|
||||
# Calculate angle between eyes
|
||||
eye_angle = np.degrees(np.arctan2(right_eye_center[1] - left_eye_center[1],
|
||||
right_eye_center[0] - left_eye_center[0]))
|
||||
# Calculate the desired eye positions in the output image
|
||||
left_eye_target = np.array([resize_size * left_eye_pos[0], resize_size * left_eye_pos[1]])
|
||||
right_eye_target = np.array([resize_size * (1.0 - left_eye_pos[0]), resize_size * left_eye_pos[1]])
|
||||
|
||||
# Calculate rotation center (midpoint between eyes)
|
||||
center = ((left_eye_center[0] + right_eye_center[0]) / 2,
|
||||
(left_eye_center[1] + right_eye_center[1]) / 2)
|
||||
# Calculate the angle between the current eye line and the target eye line
|
||||
current_angle = np.degrees(np.arctan2(right_eye_center[1] - left_eye_center[1],
|
||||
right_eye_center[0] - left_eye_center[0]))
|
||||
target_angle = np.degrees(np.arctan2(right_eye_target[1] - left_eye_target[1],
|
||||
right_eye_target[0] - left_eye_target[0]))
|
||||
rotation_angle = target_angle - current_angle
|
||||
|
||||
# Create rotation matrix
|
||||
rotation_matrix = cv2.getRotationMatrix2D(center, eye_angle, 1.0)
|
||||
# Calculate the scale factor to match the desired eye distance
|
||||
current_eye_distance = np.linalg.norm(right_eye_center - left_eye_center)
|
||||
target_eye_distance = np.linalg.norm(right_eye_target - left_eye_target)
|
||||
scale = target_eye_distance / current_eye_distance
|
||||
|
||||
# Calculate translation to position left eye
|
||||
tx = resize_size * left_eye_pos[0] - left_eye_center[0]
|
||||
ty = resize_size * left_eye_pos[1] - left_eye_center[1]
|
||||
rotation_matrix[0, 2] += tx
|
||||
rotation_matrix[1, 2] += ty
|
||||
# Create the transformation matrix
|
||||
# First, translate to origin (center of eyes)
|
||||
center = np.array([(left_eye_center[0] + right_eye_center[0]) / 2,
|
||||
(left_eye_center[1] + right_eye_center[1]) / 2])
|
||||
M1 = np.array([[1, 0, -center[0]],
|
||||
[0, 1, -center[1]],
|
||||
[0, 0, 1]])
|
||||
|
||||
# Convert image to numpy array for OpenCV processing
|
||||
img_np = np.array(image)
|
||||
img_np = cv2.cvtColor(img_np, cv2.COLOR_RGB2BGR)
|
||||
# Then rotate
|
||||
angle_rad = np.radians(rotation_angle)
|
||||
M2 = np.array([[np.cos(angle_rad), -np.sin(angle_rad), 0],
|
||||
[np.sin(angle_rad), np.cos(angle_rad), 0],
|
||||
[0, 0, 1]])
|
||||
|
||||
# Then scale
|
||||
M3 = np.array([[scale, 0, 0],
|
||||
[0, scale, 0],
|
||||
[0, 0, 1]])
|
||||
|
||||
# Finally, translate to target position
|
||||
target_center = np.array([(left_eye_target[0] + right_eye_target[0]) / 2,
|
||||
(left_eye_target[1] + right_eye_target[1]) / 2])
|
||||
M4 = np.array([[1, 0, target_center[0]],
|
||||
[0, 1, target_center[1]],
|
||||
[0, 0, 1]])
|
||||
|
||||
# Combine all transformations
|
||||
M = M4 @ M3 @ M2 @ M1
|
||||
|
||||
# Convert to 2x3 matrix for OpenCV
|
||||
rotation_matrix = M[:2, :]
|
||||
|
||||
# Apply transformation
|
||||
aligned_face = cv2.warpAffine(img_np, rotation_matrix, (resize_size, resize_size),
|
||||
|
|
@ -421,8 +479,6 @@ def crop_and_align_face(image, face_data, resize_size, face_resolution_threshold
|
|||
aligned_face = cv2.cvtColor(aligned_face, cv2.COLOR_BGR2RGB)
|
||||
aligned_face = Image.fromarray(aligned_face)
|
||||
|
||||
# draw_landmarks(img_np, shape, (x1, y1, x2, y2), "final/aligned_face.jpg")
|
||||
|
||||
return aligned_face
|
||||
|
||||
except Exception as e:
|
||||
|
|
|
|||
Loading…
Reference in a new issue