Update app.py

app.py

@@ -1,143 +1,110 @@
-import os
-os.system("pip install cmake")
-os.system("pip install dlib opencv-python numpy Pillow gradio")
-
 import gradio as gr
 import cv2
 import numpy as np
 from PIL import Image
-import dlib
-
-# --- Load models ---
-detector = dlib.get_frontal_face_detector()
-predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")  # Requires download
-
-# --- Helpers ---
-def load_image(file):
-    img = cv2.imread(file.name)
-    if img is None:
-        raise ValueError(f"Failed to load image: {file.name}")
-    return img
-
-def get_landmarks(img):
-    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
-    faces = detector(gray)
-    if len(faces) == 0:
-        raise Exception("No face detected.")
-    shape = predictor(gray, faces[0])
-    return np.array([[p.x, p.y] for p in shape.parts()], np.int32)
-
-def apply_affine_transform(src, src_tri, dst_tri, size):
-    warp_mat = cv2.getAffineTransform(np.float32(src_tri), np.float32(dst_tri))
-    dst = cv2.warpAffine(src, warp_mat, (size[0], size[1]), None, flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT_101)
-    return dst
-
-def morph_triangle(img1, img2, img, t1, t2, t, alpha):
-    r1 = cv2.boundingRect(np.float32([t1]))
-    r2 = cv2.boundingRect(np.float32([t2]))
-    r = cv2.boundingRect(np.float32([t]))
-    
-    t1_rect = []
-    t2_rect = []
-    t_rect = []
-
-    for i in range(3):
-        t1_rect.append(((t1[i][0] - r1[0]),(t1[i][1] - r1[1])))
-        t2_rect.append(((t2[i][0] - r2[0]),(t2[i][1] - r2[1])))
-        t_rect.append(((t[i][0] - r[0]),(t[i][1] - r[1])))
-
-    img1_rect = img1[r1[1]:r1[1]+r1[3], r1[0]:r1[0]+r1[2]]
-    img2_rect = img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]]
-
-    warp_img1 = apply_affine_transform(img1_rect, t1_rect, t_rect, (r[2], r[3]))
-    warp_img2 = apply_affine_transform(img2_rect, t2_rect, t_rect, (r[2], r[3]))
-
-    img_rect = (1.0 - alpha) * warp_img1 + alpha * warp_img2
-
-    mask = np.zeros((r[3], r[2], 3), dtype=np.float32)
-    cv2.fillConvexPoly(mask, np.int32(t_rect), (1.0, 1.0, 1.0), 16, 0)
-    img[r[1]:r[1]+r[3], r[0]:r[0]+r[2]] = img[r[1]:r[1]+r[3], r[0]:r[0]+r[2]] * (1 - mask) + img_rect * mask
-
-def delaunay_triangulation(points, w, h):
-    subdiv = cv2.Subdiv2D((0, 0, w, h))
-    for p in points:
+import mediapipe as mp
+
+# MediaPipe face mesh setup
+mp_face_mesh = mp.solutions.face_mesh
+face_mesh = mp_face_mesh.FaceMesh(static_image_mode=True)
+
+# Get facial landmarks
+def get_landmarks(image):
+    rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+    results = face_mesh.process(rgb)
+    h, w, _ = image.shape
+
+    if results.multi_face_landmarks:
+        landmarks = []
+        for pt in results.multi_face_landmarks[0].landmark:
+            x, y = int(pt.x * w), int(pt.y * h)
+            landmarks.append((x, y))
+        return np.array(landmarks, dtype=np.int32)
+    return None
+
+# Morph images based on landmarks and alpha
+def morph_images(img1, img2, alpha):
+    lm1 = get_landmarks(img1)
+    lm2 = get_landmarks(img2)
+
+    if lm1 is None or lm2 is None:
+        return Image.fromarray(img1)  # fallback
+
+    lm_avg = ((1 - alpha) * lm1 + alpha * lm2).astype(np.int32)
+
+    # Triangulation
+    rect = (0, 0, img1.shape[1], img1.shape[0])
+    subdiv = cv2.Subdiv2D(rect)
+    for p in lm_avg:
         subdiv.insert((p[0], p[1]))
-    triangle_list = subdiv.getTriangleList()
-    triangles = []
-    for t in triangle_list:
-        pts = [(int(t[0]), int(t[1])), (int(t[2]), int(t[3])), (int(t[4]), int(t[5]))]
-        idx = []
-        for pt in pts:
-            for i, p in enumerate(points):
-                if np.linalg.norm(np.array(pt) - p) < 1.0:
-                    idx.append(i)
-        if len(idx) == 3:
-            triangles.append(tuple(idx))
-    return triangles
-
-def morph_faces(img1, img2, alpha=0.5):
-    img1 = np.float32(img1)
-    img2 = np.float32(img2)
-    points1 = get_landmarks(img1)
-    points2 = get_landmarks(img2)
-    points = []
-    for i in range(len(points1)):
-        x = (1 - alpha) * points1[i][0] + alpha * points2[i][0]
-        y = (1 - alpha) * points1[i][1] + alpha * points2[i][1]
-        points.append((int(x), int(y)))
-
-    morphed = np.zeros(img1.shape, dtype=img1.dtype)
-    tri = delaunay_triangulation(points, img1.shape[1], img1.shape[0])
-
-    for t in tri:
-        t1 = [points1[t[0]], points1[t[1]], points1[t[2]]]
-        t2 = [points2[t[0]], points2[t[1]], points2[t[2]]]
-        t_ = [points[t[0]], points[t[1]], points[t[2]]]
-        morph_triangle(img1, img2, morphed, t1, t2, t_, alpha)
+    triangles = subdiv.getTriangleList().astype(np.int32)
+
+    def get_indices(tri, points):
+        idxs = []
+        for p in tri:
+            for i, pt in enumerate(points):
+                if np.linalg.norm(np.array(p) - np.array(pt)) < 1.0:
+                    idxs.append(i)
+        return idxs if len(idxs) == 3 else None
+
+    morphed = np.zeros_like(img1)
+    for tri in triangles:
+        pts = [(tri[0], tri[1]), (tri[2], tri[3]), (tri[4], tri[5])]
+        idxs = get_indices(pts, lm_avg.tolist())
+        if idxs is None: continue
+
+        t1 = np.float32([lm1[i] for i in idxs])
+        t2 = np.float32([lm2[i] for i in idxs])
+        t = np.float32([lm_avg[i] for i in idxs])
+
+        def warp_triangle(src, t_src, t_dst):
+            rect_src = cv2.boundingRect(t_src)
+            rect_dst = cv2.boundingRect(t_dst)
+
+            t_src_offset = np.array([[pt[0]-rect_src[0], pt[1]-rect_src[1]] for pt in t_src], np.float32)
+            t_dst_offset = np.array([[pt[0]-rect_dst[0], pt[1]-rect_dst[1]] for pt in t_dst], np.float32)
+
+            mask = np.zeros((rect_dst[3], rect_dst[2], 3), dtype=np.float32)
+            cv2.fillConvexPoly(mask, np.int32(t_dst_offset), (1.0, 1.0, 1.0), 16, 0)
+
+            src_crop = src[rect_src[1]:rect_src[1]+rect_src[3], rect_src[0]:rect_src[0]+rect_src[2]]
+            warp_mat = cv2.getAffineTransform(t_src_offset, t_dst_offset)
+            dst_crop = cv2.warpAffine(src_crop, warp_mat, (rect_dst[2], rect_dst[3]), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT_101)
+            morphed[rect_dst[1]:rect_dst[1]+rect_dst[3], rect_dst[0]:rect_dst[0]+rect_dst[2]] *= (1 - mask)
+            morphed[rect_dst[1]:rect_dst[1]+rect_dst[3], rect_dst[0]:rect_dst[0]+rect_dst[2]] += dst_crop * mask
+
+        warp_triangle(img1, t1, t)
+        warp_triangle(img2, t2, t)
 
     return Image.fromarray(np.uint8(morphed))
 
-# --- Gradio Function ---
-def lipsync_interface(mm, aa, ee, oo, ww, na, s_aa, s_oo, s_ee, s_ww, s_na):
-    mm_img = load_image(mm)
-    result = mm_img.copy()
-
-    if s_aa > 0.0:
-        aa_img = load_image(aa)
-        result = np.array(morph_faces(result, aa_img, s_aa))
-    if s_oo > 0.0:
-        oo_img = load_image(oo)
-        result = np.array(morph_faces(result, oo_img, s_oo))
-    if s_ee > 0.0:
-        ee_img = load_image(ee)
-        result = np.array(morph_faces(result, ee_img, s_ee))
-    if s_ww > 0.0:
-        ww_img = load_image(ww)
-        result = np.array(morph_faces(result, ww_img, s_ww))
-    if s_na > 0.0:
-        na_img = load_image(na)
-        result = np.array(morph_faces(result, na_img, s_na))
-
-    return Image.fromarray(result)
-
-# --- Gradio UI ---
+def process(mm_image, aa_image, ee_image, oo_image, ww_image, na_image, slider_aa, slider_oo, slider_ee, slider_ww, slider_na):
+    def load(img): return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
+
+    img = load(mm_image)
+    for s, phoneme in [(slider_aa, aa_image), (slider_oo, oo_image), (slider_ee, ee_image), (slider_ww, ww_image), (slider_na, na_image)]:
+        if s > 0.0:
+            target = load(phoneme)
+            img = np.array(morph_images(img, target, s))
+
+    return Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
+
 iface = gr.Interface(
-    fn=lipsync_interface,
+    fn=process,
     inputs=[
-        gr.Image(label="MM Image (Neutral)", type="file"),
-        gr.Image(label="AA Image", type="file"),
-        gr.Image(label="EE Image", type="file"),
-        gr.Image(label="OO Image", type="file"),
-        gr.Image(label="WW Image", type="file"),
-        gr.Image(label="NA Image", type="file"),
-
-        gr.Slider(0.0, 1.0, step=0.05, label="Morph Strength AA"),
-        gr.Slider(0.0, 1.0, step=0.05, label="Morph Strength OO"),
-        gr.Slider(0.0, 1.0, step=0.05, label="Morph Strength EE"),
-        gr.Slider(0.0, 1.0, step=0.05, label="Morph Strength WW"),
-        gr.Slider(0.0, 1.0, step=0.05, label="Morph Strength NA"),
+        gr.Image(label="MM Image (Neutral)"),
+        gr.Image(label="AA Image"),
+        gr.Image(label="EE Image"),
+        gr.Image(label="OO Image"),
+        gr.Image(label="WW Image"),
+        gr.Image(label="NA Image"),
+        gr.Slider(0, 1, 0.05, label="Strength AA"),
+        gr.Slider(0, 1, 0.05, label="Strength OO"),
+        gr.Slider(0, 1, 0.05, label="Strength EE"),
+        gr.Slider(0, 1, 0.05, label="Strength WW"),
+        gr.Slider(0, 1, 0.05, label="Strength NA"),
     ],
-    outputs=gr.Image(label="Interpolated Lip-sync Frame"),
+    outputs=gr.Image(label="Lipsynced Output"),
     live=True
 )