demo v1

.gitignore

@@ -0,0 +1 @@
+Demo/flagged/

app.py

@@ -0,0 +1,137 @@
+import gradio as gr
+import torch
+from src.predict import predict_from_video
+from src.islr.islr_model import DummyISLRModel
+
+
+#device = 'cpu'
+#device = 'cuda'
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+# Simulación de diccionario con paths
+dataset_models = {
+    "PERU":  {"path":"models/demo_model.pt","num_classes":100},
+    "WLASL": {"path":"models/demo_model.pt","num_classes":100},
+}
+
+# Diccionario de rutas y etiquetas por dataset
+dataset_examples = {
+    "PERU": [
+        {"label": "📘 **Glosa: `libro`**", "path": "videos/wlasl/book.mp4"},
+        {"label": "🏠 **Glosa: `casa`**", "path": "videos/wlasl/book.mp4"},
+        {"label": "📘 **Glosa: `libro2`**", "path": "videos/wlasl/book.mp4"},
+        {"label": "🏠 **Glosa: `casa2`**", "path": "videos/wlasl/book.mp4"},
+    ],
+    "WLASL": [
+        {"label": "📙 **Glosa: `read`**", "path":"videos/wlasl/book.mp4"},
+        {"label": "🏫 **Glosa: `school`**", "path":"videos/wlasl/book.mp4"},
+        {"label": "📙 **Glosa: `read2`**", "path":"videos/wlasl/book.mp4"},
+        {"label": "🏫 **Glosa: `school2`**", "path":"videos/wlasl/book.mp4"},
+    ]
+}
+
+# === Carga el modelo según el dataset seleccionado ===
+def load_model_and_examples(dataset):
+    model_path  = dataset_models.get(dataset)['path']
+    num_classes = dataset_models.get(dataset)['num_classes']
+    model = DummyISLRModel(num_classes=num_classes)
+    model.load_state_dict(torch.load(model_path, map_location=device))
+    model.eval()
+    print(f"Model {dataset} Loaded!")
+    examples = dataset_examples.get(dataset, [{"label": "", "path": ""}, {"label": "", "path": ""}])
+    return (
+        model,
+        gr.update(visible=True),
+        gr.update(value=examples[0]["path"]),
+        examples[0]["path"],
+        gr.update(value=examples[0]["label"]),
+        gr.update(value=examples[1]["path"]),
+        examples[1]["path"],
+        gr.update(value=examples[1]["label"]),
+        gr.update(value=examples[2]["path"]),
+        examples[2]["path"],
+        gr.update(value=examples[2]["label"]),
+        gr.update(value=examples[3]["path"]),
+        examples[3]["path"],
+        gr.update(value=examples[3]["label"]),
+        gr.update(interactive=True)  # activa el botón
+    )
+
+# === Usamos el modelo cargado en el State ===
+def classify_video_with_model(video, model):
+    top1, top5_df = predict_from_video(video, model=model)  # asegúrate de pasar el modelo en `predict_from_video`
+    return f"Top-1: {top1}", top5_df
+
+with gr.Blocks() as demo:
+    gr.Markdown("# 🧠 ISLR Demo con Mediapipe y 100 Clases")
+    gr.Markdown("Sube un video o usa la webcam. El modelo clasificará la seña y mostrará las 5 clases más probables.")
+
+    # === Selector de dataset
+    gr.Markdown("## 📁 Filtrar por Language")
+    dataset_selector = gr.Dropdown(choices=list(dataset_examples.keys()), value=None, label="Selecciona el lenguaje")
+
+
+    # === Estado del modelo ===
+    current_model = gr.State()
+    video_path_1 = gr.State()
+    video_path_2 = gr.State()
+    video_path_3 = gr.State()
+    video_path_4 = gr.State()
+
+    # === Entrada de video + salida
+    with gr.Row():
+        video_input = gr.Video(sources=["upload", "webcam"], label="🎥 Video de entrada", width=300, height=400)
+        with gr.Column():
+            output_text = gr.Text(label="Predicción Top-1")
+            output_table = gr.Label(num_top_classes=5)
+            button_classify = gr.Button("🔍 Clasificar",interactive=False)
+            
+            button_classify.click(
+                fn=classify_video_with_model,
+                inputs=[video_input, current_model],
+                outputs=[output_text, output_table]
+            )
+
+
+
+
+    # === Contenedor dinámico de ejemplos
+    examples_output = gr.Column(visible=True)
+
+    with examples_output:
+        with gr.Row():
+            with gr.Column(scale=1, min_width=100):
+                m1 = gr.Markdown("📘 **Glosa: **")
+                v1 = gr.Video(interactive=False, width=160, height=120)
+                b1 = gr.Button("Usar", scale=0)
+            with gr.Column(scale=1, min_width=100):
+                m2 = gr.Markdown("🏠 **Glosa: **")
+                v2 = gr.Video(interactive=False, width=160, height=120)
+                b2 = gr.Button("Usar", scale=0)
+            with gr.Column(scale=1, min_width=100):
+                m3 = gr.Markdown("🏠 **Glosa: **")
+                v3 = gr.Video(interactive=False, width=160, height=120)
+                b3 = gr.Button("Usar", scale=0)
+            with gr.Column(scale=1, min_width=100):
+                m4 = gr.Markdown("🏠 **Glosa: **")
+                v4 = gr.Video(interactive=False, width=160, height=120)
+                b4 = gr.Button("Usar", scale=0)
+
+        b1.click(fn=lambda path: path, inputs=video_path_1, outputs=video_input)
+        b2.click(fn=lambda path: path, inputs=video_path_2, outputs=video_input)
+        b3.click(fn=lambda path: path, inputs=video_path_3, outputs=video_input)
+        b4.click(fn=lambda path: path, inputs=video_path_4, outputs=video_input)
+
+    gr.Markdown("## 📁 Ejemplos de videos")
+    # === Al cambiar dataset, cargamos modelo + ejemplos
+    dataset_selector.change(
+        fn=load_model_and_examples,
+        inputs=dataset_selector,
+        outputs=[current_model, examples_output, v1,video_path_1,m1, v2, video_path_2, m2, v3, video_path_3, m3, v4, video_path_4, m4,
+                 button_classify
+            ]
+    )
+
+if __name__ == "__main__":
+    demo.launch(server_port=8080)

requirements.txt

@@ -0,0 +1,6 @@
+gradio
+torch
+mediapipe
+opencv-python
+numpy
+pandas

src/islr/__init__.py

@@ -0,0 +1 @@
+from . import *

src/islr/islr_model.py

@@ -0,0 +1,13 @@
+import torch.nn as nn
+
+class DummyISLRModel(nn.Module):
+    def __init__(self, input_dim=225, num_classes=100):
+        super().__init__()
+        self.fc = nn.Sequential(
+            nn.Linear(input_dim, 128),
+            nn.ReLU(),
+            nn.Linear(128, num_classes)
+        )
+
+    def forward(self, x):
+        return self.fc(x)

src/islr/save_dummy_model.py

@@ -0,0 +1,12 @@
+import torch
+from islr_model import DummyISLRModel
+
+model = DummyISLRModel(num_classes=100)
+for param in model.parameters():
+    if param.dim() > 1:
+        torch.nn.init.xavier_uniform_(param)
+    else:
+        torch.nn.init.zeros_(param)
+
+torch.save(model.state_dict(), "demo_model.pt")
+print("✅ Modelo guardado como demo_model.pt")

src/pose/__init__.py

@@ -0,0 +1 @@
+from . import *

src/pose/keypoints_utils.py

@@ -0,0 +1,15 @@
+import mediapipe as mp
+import cv2
+import numpy as np
+
+mp_holistic = mp.solutions.holistic
+
+def extract_keypoints_from_frame(frame):
+    with mp_holistic.Holistic(static_image_mode=True) as holistic:
+        results = holistic.process(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+        if results.pose_landmarks and results.left_hand_landmarks and results.right_hand_landmarks:
+            pose = np.array([[p.x, p.y, p.z] for p in results.pose_landmarks.landmark])
+            left = np.array([[p.x, p.y, p.z] for p in results.left_hand_landmarks.landmark])
+            right = np.array([[p.x, p.y, p.z] for p in results.right_hand_landmarks.landmark])
+            return np.concatenate([pose, left, right], axis=0).flatten()
+        return np.random.random(33*3 + 21*3*2)  # fallback

src/predict.py

@@ -0,0 +1,47 @@
+import cv2
+import torch
+import numpy as np
+import pandas as pd
+from .islr.islr_model import DummyISLRModel
+from .pose.keypoints_utils import extract_keypoints_from_frame
+
+#model = DummyISLRModel(num_classes=100)
+#model.load_state_dict(torch.load("demo_model.pt", map_location='cpu'))
+#model.eval()
+
+LABELS = [f"Clase {i}" for i in range(100)]
+
+def predict_from_video(video_path,model=None):
+    cap = cv2.VideoCapture(video_path)
+    keypoints = []
+
+    while cap.isOpened():
+        ret, frame = cap.read()
+        if not ret:
+            break
+        keypoint = extract_keypoints_from_frame(frame)
+        keypoints.append(keypoint)
+
+    cap.release()
+    if not keypoints:
+        return "No keypoints detected", pd.DataFrame()
+
+    x = torch.tensor(np.mean(keypoints, axis=0)).float().unsqueeze(0)
+    #print("x:")
+    #print(x)
+    with torch.no_grad():
+        logits = model(x)
+        probs = torch.softmax(logits, dim=1).numpy()[0]
+
+    #print("probs:")
+    #print(probs)
+    top5_idx = probs.argsort()[-5:][::-1]
+
+    top5_labels = [LABELS[i] for i in top5_idx]
+    top5_probs = [probs[i] for i in top5_idx]
+    confidences = {LABELS[i]: float(probs[i]) for i in top5_idx}    
+
+    #print("confidences:")
+    #print(confidences)
+    #df = pd.DataFrame({"label": top5_labels, "value": top5_probs})
+    return top5_labels[0],confidences # df

src/simple_demo.py

@@ -0,0 +1,24 @@
+import gradio as gr
+from predict import predict_from_video
+
+def classify_video(video):
+    top1, top5_df = predict_from_video(video)
+    return f"Top-1: {top1}", top5_df
+
+demo = gr.Interface(
+    fn=classify_video,
+    inputs=gr.Video(sources=["upload", "webcam"], label="🎥 Video (webcam o archivo)"),
+    outputs=[
+        gr.Text(label="Predicción principal"),
+        gr.Label(num_top_classes=5),
+    ],
+    #outputs="text",
+    title="🧠 ISLR Demo con Mediapipe y 100 Clases",
+    description="Clasificador de lenguaje de señas aislado. Muestra las Top-5 clases más probables.",
+    #examples=["/home/va0831/slr/SLR_2024/Gloss/SignLanguageRecognition/J7tP98oDxqE_000000_000066-msasl-book.mp4",
+    #          "/home/va0831/slr/SLR_2024/Gloss/SignLanguageRecognition/J7tP98oDxqE_000000_000066-msasl-book.mp4"],
+    #example_labels = ["book","house"],
+)
+
+if __name__ == "__main__":
+    demo.launch(server_port=9090)