Create useHandDetection.js

hooks/useHandDetection.js

@@ -0,0 +1,238 @@
+import { useState, useEffect, useRef } from 'react';
+import { HandLandmarker, FilesetResolver } from '@mediapipe/tasks-vision';
+import { drawLandmarks, analyzeHandGesture } from '../utils/handUtils';
+
+const useHandDetection = (videoRef, canvasRef, isMobile) => {
+  const [handLandmarker, setHandLandmarker] = useState(null);
+  const [handDetected, setHandDetected] = useState(false);
+  const [cardsDetected, setCardsDetected] = useState(false);
+  const [detectedCards, setDetectedCards] = useState([]);
+  const [isMouthOpen, setIsMouthOpen] = useState(false);
+  const [isLeftHand, setIsLeftHand] = useState(true);
+  const [thumbPosition, setThumbPosition] = useState({ x: 0, y: 0 });
+  const [isFirstLoad, setIsFirstLoad] = useState(true);
+  const [detectionMode, setDetectionMode] = useState('both'); // 'cards', 'hand', 'both'
+  
+  const requestRef = useRef(null);
+  const lastDetectionTimeRef = useRef(0);
+  const lastCardDetectionTimeRef = useRef(0);
+  const isComponentMounted = useRef(true);
+  
+  // Card detection using canvas analysis or Gemini Vision API
+  const detectPlayingCards = async (canvas) => {
+    try {
+      // Canvas를 이미지로 변환
+      const imageData = canvas.toDataURL('image/jpeg', 0.8);
+      
+      // Gemini Vision API를 사용하여 카드 감지
+      const response = await fetch('/api/gemini', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          image: imageData,
+          prompt: 'Detect any playing cards in this image. List the cards you see (suit and rank). If no cards are visible, respond with "NO_CARDS".',
+          mode: 'card_detection'
+        }),
+      });
+      
+      if (response.ok) {
+        const data = await response.json();
+        if (data.result && data.result !== 'NO_CARDS') {
+          return {
+            detected: true,
+            cards: data.result
+          };
+        }
+      }
+      
+      return { detected: false, cards: [] };
+    } catch (error) {
+      console.error('Error detecting cards:', error);
+      return { detected: false, cards: [] };
+    }
+  };
+  
+  // Initialize the HandLandmarker
+  useEffect(() => {
+    isComponentMounted.current = true;
+    
+    const initializeHandLandmarker = async () => {
+      try {
+        const vision = await FilesetResolver.forVisionTasks(
+          "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
+        );
+        
+        if (!isComponentMounted.current) return;
+        
+        const landmarker = await HandLandmarker.createFromOptions(vision, {
+          baseOptions: {
+            modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/latest/hand_landmarker.task",
+            delegate: "GPU"
+          },
+          runningMode: "VIDEO",
+          numHands: 1,
+          minHandDetectionConfidence: 0.5,
+          minHandPresenceConfidence: 0.5,
+          minTrackingConfidence: 0.5
+        });
+        
+        if (!isComponentMounted.current) return;
+        
+        setHandLandmarker(landmarker);
+        console.log("Hand landmarker initialized successfully");
+        
+        // Set first load to false after initialization
+        setTimeout(() => {
+          if (isComponentMounted.current) {
+            setIsFirstLoad(false);
+          }
+        }, 3000);
+      } catch (error) {
+        console.error("Error initializing hand landmarker:", error);
+      }
+    };
+
+    initializeHandLandmarker();
+
+    return () => {
+      isComponentMounted.current = false;
+      if (requestRef.current) {
+        cancelAnimationFrame(requestRef.current);
+        requestRef.current = null;
+      }
+    };
+  }, []);
+
+  // Process video frames and detect hand gestures and cards
+  useEffect(() => {
+    if (!handLandmarker || !videoRef.current || !canvasRef.current) return;
+
+    const video = videoRef.current;
+    const canvas = canvasRef.current;
+    const ctx = canvas.getContext('2d');
+
+    const detectFrame = async (now) => {
+      if (!isComponentMounted.current) return;
+      
+      if (video.readyState < 2) {
+        requestRef.current = requestAnimationFrame(detectFrame);
+        return;
+      }
+
+      // Clear the canvas
+      ctx.clearRect(0, 0, canvas.width, canvas.height);
+      
+      // Draw the video frame on the canvas
+      const videoWidth = video.videoWidth;
+      const videoHeight = video.videoHeight;
+      
+      let drawWidth = canvas.width;
+      let drawHeight = canvas.height;
+      let offsetX = 0;
+      let offsetY = 0;
+      
+      ctx.drawImage(video, offsetX, offsetY, drawWidth, drawHeight);
+      
+      // PRIORITY: Detect playing cards first (every 500ms)
+      if (now - lastCardDetectionTimeRef.current > 500) {
+        lastCardDetectionTimeRef.current = now;
+        
+        const cardResult = await detectPlayingCards(canvas);
+        setCardsDetected(cardResult.detected);
+        setDetectedCards(cardResult.cards);
+        
+        // If cards are detected, we can skip hand detection or make it secondary
+        if (cardResult.detected) {
+          setDetectionMode('cards');
+          
+          // Draw card detection overlay
+          ctx.strokeStyle = '#00ff00';
+          ctx.lineWidth = 3;
+          ctx.font = '20px Arial';
+          ctx.fillStyle = '#00ff00';
+          ctx.fillText('🎴 Playing Cards Detected!', 10, 30);
+        } else {
+          setDetectionMode('hand');
+        }
+      }
+      
+      // Only detect hands if no cards are detected or in 'both' mode
+      if (!cardsDetected || detectionMode === 'both') {
+        // Hand detection (every 100ms)
+        if (now - lastDetectionTimeRef.current > 100) {
+          lastDetectionTimeRef.current = now;
+          
+          const results = handLandmarker.detectForVideo(video, now);
+          
+          // Check if hands are detected
+          if (results.landmarks && results.landmarks.length > 0) {
+            const landmarks = results.landmarks[0];
+            setHandDetected(true);
+            
+            // Draw hand landmarks
+            drawLandmarks(ctx, landmarks, canvas, isMobile);
+            
+            // Analyze hand gesture
+            const { isOpen, isLeftHand: isLeft, thumbPosition: thumbPos } = analyzeHandGesture(landmarks);
+            
+            // Update state with hand information
+            setIsLeftHand(isLeft);
+            setThumbPosition({
+              x: thumbPos.x * canvas.width,
+              y: thumbPos.y * canvas.height
+            });
+            
+            // Update UI based on hand state
+            if (isOpen !== isMouthOpen) {
+              setIsMouthOpen(isOpen);
+            }
+          } else {
+            // No hands detected
+            setHandDetected(false);
+            if (isMouthOpen) {
+              setIsMouthOpen(false);
+            }
+          }
+        }
+      }
+      
+      // Draw detection status
+      ctx.font = '14px Arial';
+      ctx.fillStyle = '#ffffff';
+      ctx.strokeStyle = '#000000';
+      ctx.lineWidth = 3;
+      const statusText = cardsDetected ? '🎴 Cards Priority Mode' : '✋ Hand Detection Mode';
+      ctx.strokeText(statusText, 10, canvas.height - 20);
+      ctx.fillText(statusText, 10, canvas.height - 20);
+      
+      requestRef.current = requestAnimationFrame(detectFrame);
+    };
+
+    requestRef.current = requestAnimationFrame(detectFrame);
+
+    return () => {
+      if (requestRef.current) {
+        cancelAnimationFrame(requestRef.current);
+        requestRef.current = null;
+      }
+    };
+  }, [handLandmarker, isMouthOpen, isMobile, videoRef, canvasRef, cardsDetected, detectionMode]);
+
+  return {
+    handDetected,
+    cardsDetected,
+    detectedCards,
+    isMouthOpen,
+    isLeftHand,
+    thumbPosition,
+    isFirstLoad,
+    isComponentMounted,
+    detectionMode,
+    // 우선순위 기반 감지 상태 반환
+    primaryDetection: cardsDetected ? 'cards' : (handDetected ? 'hand' : 'none')
+  };
+};
+
+export default useHandDetection;