import { useState, useEffect, useRef } from "react"; import { Container, Paper, Button, Box, Typography, LinearProgress, Chip, IconButton, Tooltip, } from "@mui/material"; import SaveOutlinedIcon from "@mui/icons-material/SaveOutlined"; import RestartAltIcon from "@mui/icons-material/RestartAlt"; import axios from "axios"; import { ComicLayout } from "../../layouts/ComicLayout"; import { getNextPanelDimensions, groupSegmentsIntoLayouts, } from "../../layouts/utils"; import { LAYOUTS } from "../../layouts/config"; import html2canvas from "html2canvas"; import { useConversation } from "@11labs/react"; // Get API URL from environment or default to localhost in development const isHFSpace = window.location.hostname.includes("hf.space"); const API_URL = isHFSpace ? "" // URL relative pour HF Spaces : import.meta.env.VITE_API_URL || "http://localhost:8000"; // Generate a unique client ID const CLIENT_ID = `client_${Math.random().toString(36).substring(2)}`; // Constants const AGENT_ID = "2MF9st3s1mNFbX01Y106"; const WS_URL = import.meta.env.VITE_WS_URL || "ws://localhost:8000/ws"; // Create axios instance with default config const api = axios.create({ headers: { "x-client-id": CLIENT_ID, }, // Ajouter baseURL pour HF Spaces ...(isHFSpace && { baseURL: window.location.origin, }), }); // Function to convert text with ** to Chip elements const formatTextWithBold = (text, isInPanel = false) => { if (!text) return ""; const parts = text.split(/(\*\*.*?\*\*)/g); return parts.map((part, index) => { if (part.startsWith("**") && part.endsWith("**")) { // Remove the ** and wrap in Chip return ( ); } return part; }); }; function App() { const [storySegments, setStorySegments] = useState([]); const [currentChoices, setCurrentChoices] = useState([]); const [isLoading, setIsLoading] = useState(false); const [isDebugMode, setIsDebugMode] = useState(false); const [isRecording, setIsRecording] = useState(false); const [wsConnected, setWsConnected] = useState(false); const audioRef = useRef(new Audio()); const comicContainerRef = useRef(null); const narrationAudioRef = useRef(new Audio()); // Separate audio ref for narration const wsRef = useRef(null); const mediaRecorderRef = useRef(null); const audioChunksRef = useRef([]); // Start the story on first render useEffect(() => { handleStoryAction("restart"); }, []); // Empty dependency array for first render only // Only setup WebSocket connection with server useEffect(() => { const setupWebSocket = () => { wsRef.current = new WebSocket(WS_URL); wsRef.current.onopen = () => { console.log('Server WebSocket connected'); setWsConnected(true); }; wsRef.current.onclose = (event) => { const reason = event.reason || 'No reason provided'; const code = event.code; console.log(`Server WebSocket disconnected - Code: ${code}, Reason: ${reason}`); console.log('Attempting to reconnect in 3 seconds...'); setWsConnected(false); // Attempt to reconnect after 3 seconds setTimeout(setupWebSocket, 3000); }; wsRef.current.onmessage = async (event) => { const data = JSON.parse(event.data); if (data.type === 'audio') { // Stop any ongoing narration if (narrationAudioRef.current) { narrationAudioRef.current.pause(); narrationAudioRef.current.currentTime = 0; } // Play the conversation audio response const audioBlob = await fetch(`data:audio/mpeg;base64,${data.audio}`).then(r => r.blob()); const audioUrl = URL.createObjectURL(audioBlob); audioRef.current.src = audioUrl; await audioRef.current.play(); } }; }; setupWebSocket(); return () => { if (wsRef.current) { wsRef.current.close(); } }; }, []); const conversation = useConversation({ agentId: AGENT_ID, onResponse: async (response) => { if (response.type === 'audio') { // Play the conversation audio response const audioBlob = new Blob([response.audio], { type: 'audio/mpeg' }); const audioUrl = URL.createObjectURL(audioBlob); audioRef.current.src = audioUrl; await audioRef.current.play(); } }, clientTools: { make_decision: async ({ decision }) => { console.log('AI made decision:', decision); // End the ElevenLabs conversation await conversation.endSession(); setIsConversationMode(false); setIsRecording(false); // Handle the choice and generate next story part await handleChoice(parseInt(decision)); } } }); const { isSpeaking } = conversation; const [isConversationMode, setIsConversationMode] = useState(false); // Audio recording setup const startRecording = async () => { try { // Stop narration audio if it's playing if (narrationAudioRef.current) { narrationAudioRef.current.pause(); narrationAudioRef.current.currentTime = 0; } // Also stop any conversation audio if playing if (audioRef.current) { audioRef.current.pause(); audioRef.current.currentTime = 0; } if (!isConversationMode) { // If we're not in conversation mode, this is the first recording setIsConversationMode(true); // Initialize ElevenLabs WebSocket connection try { // Pass available choices to the conversation const currentChoiceIds = currentChoices.map(choice => choice.id).join(','); await conversation.startSession({ agentId: AGENT_ID, initialContext: `This is the current situation : ${storySegments[storySegments.length - 1].text}. Those are the possible actions, ${currentChoices.map((choice, index) => `decision ${index + 1} : ${choice.text}`).join(', ')}.` }); console.log('ElevenLabs WebSocket connected'); } catch (error) { console.error('Error initializing ElevenLabs conversation:', error); return; } } else if (isSpeaking) { // Only handle stopping the agent if we're in conversation mode await conversation.endSession(); const wsUrl = `wss://api.elevenlabs.io/v1/convai/conversation?agent_id=${AGENT_ID}`; await conversation.startSession({ url: wsUrl }); } // Only stop narration if it's actually playing if (!isConversationMode && narrationAudioRef.current) { narrationAudioRef.current.pause(); narrationAudioRef.current.currentTime = 0; } const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); mediaRecorderRef.current = new MediaRecorder(stream); audioChunksRef.current = []; mediaRecorderRef.current.ondataavailable = (event) => { if (event.data.size > 0) { audioChunksRef.current.push(event.data); } }; mediaRecorderRef.current.onstop = async () => { const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/wav' }); const reader = new FileReader(); reader.onload = async () => { const base64Audio = reader.result.split(',')[1]; if (isConversationMode) { try { // Send audio to ElevenLabs conversation await conversation.send({ type: 'audio', data: base64Audio }); } catch (error) { console.error('Error sending audio to ElevenLabs:', error); } } else { // Otherwise use the original WebSocket connection if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) { console.log('Sending audio to server via WebSocket'); wsRef.current.send(JSON.stringify({ type: 'audio_input', audio: base64Audio, client_id: CLIENT_ID })); } } }; reader.readAsDataURL(audioBlob); }; mediaRecorderRef.current.start(); setIsRecording(true); } catch (error) { console.error('Error starting recording:', error); } }; const stopRecording = () => { if (mediaRecorderRef.current && isRecording) { mediaRecorderRef.current.stop(); setIsRecording(false); mediaRecorderRef.current.stream.getTracks().forEach(track => track.stop()); } }; const generateImagesForStory = async ( imagePrompts, segmentIndex, currentSegments ) => { try { console.log("[generateImagesForStory] Starting with:", { promptsCount: imagePrompts.length, segmentIndex, segmentsCount: currentSegments.length, }); console.log("Image prompts:", imagePrompts); console.log("Current segments:", currentSegments); let localSegments = [...currentSegments]; // Traiter chaque prompt un par un for ( let promptIndex = 0; promptIndex < imagePrompts.length; promptIndex++ ) { // Recalculer le layout actuel pour chaque image const layouts = groupSegmentsIntoLayouts(localSegments); console.log("[Layout] Current layouts:", layouts); const currentLayout = layouts[layouts.length - 1]; const layoutType = currentLayout?.type || "COVER"; console.log("[Layout] Current type:", layoutType); // Vérifier si nous avons de la place dans le layout actuel const currentSegmentImages = currentLayout.segments[currentLayout.segments.length - 1].images || []; const actualImagesCount = currentSegmentImages.filter( (img) => img !== null ).length; console.log("[Layout] Current segment images:", { total: currentSegmentImages.length, actual: actualImagesCount, hasImages: currentSegmentImages.some((img) => img !== null), currentImages: currentSegmentImages.map((img) => img ? "image" : "null" ), }); const panelDimensions = LAYOUTS[layoutType].panels[promptIndex]; console.log( "[Layout] Panel dimensions for prompt", promptIndex, ":", panelDimensions ); // Ne créer une nouvelle page que si nous avons encore des prompts à traiter // et qu'il n'y a plus de place dans le layout actuel if (!panelDimensions && promptIndex < imagePrompts.length - 1) { console.log( "[Layout] Creating new page - No space in current layout" ); // Créer un nouveau segment pour la nouvelle page const newSegment = { ...localSegments[segmentIndex], images: Array(imagePrompts.length - promptIndex).fill(null), }; localSegments = [...localSegments, newSegment]; segmentIndex = localSegments.length - 1; console.log("[Layout] New segment created:", { segmentIndex, totalSegments: localSegments.length, imagesArray: newSegment.images, }); // Mettre à jour l'état avec le nouveau segment setStorySegments(localSegments); continue; // Recommencer la boucle avec le nouveau segment } // Si nous n'avons pas de dimensions de panneau et c'est le dernier prompt, // ne pas continuer if (!panelDimensions) { console.log( "[Layout] Stopping - No more space and no more prompts to process" ); break; } console.log( `[Image] Generating image ${promptIndex + 1}/${imagePrompts.length}:`, { prompt: imagePrompts[promptIndex], dimensions: panelDimensions, } ); let retryCount = 0; const maxRetries = 3; let success = false; while (retryCount < maxRetries && !success) { try { if (retryCount > 0) { console.log( `[Image] Retry attempt ${retryCount} for image ${ promptIndex + 1 }` ); } const result = await api.post( `${API_URL}/api/generate-image-direct`, { prompt: imagePrompts[promptIndex], width: panelDimensions.width, height: panelDimensions.height, } ); console.log(`[Image] Response for image ${promptIndex + 1}:`, { success: result.data.success, hasImage: !!result.data.image_base64, imageLength: result.data.image_base64?.length, }); if (result.data.success) { console.log( `[Image] Image ${promptIndex + 1} generated successfully` ); // Mettre à jour les segments locaux const currentImages = [ ...(localSegments[segmentIndex].images || []), ]; // Remplacer le null à l'index du prompt par la nouvelle image currentImages[promptIndex] = result.data.image_base64; localSegments[segmentIndex] = { ...localSegments[segmentIndex], images: currentImages, }; console.log("[State] Updating segments with new image:", { segmentIndex, imageIndex: promptIndex, imagesArray: currentImages.map((img) => img ? "image" : "null" ), }); // Mettre à jour l'état avec les segments mis à jour setStorySegments([...localSegments]); success = true; } else { console.error( `[Image] Generation failed for image ${promptIndex + 1}:`, result.data.error ); retryCount++; if (retryCount < maxRetries) { // Attendre un peu avant de réessayer (backoff exponentiel) await new Promise((resolve) => setTimeout(resolve, 1000 * Math.pow(2, retryCount)) ); } } } catch (error) { console.error( `[Image] Error generating image ${promptIndex + 1}:`, error ); retryCount++; if (retryCount < maxRetries) { // Attendre un peu avant de réessayer (backoff exponentiel) await new Promise((resolve) => setTimeout(resolve, 1000 * Math.pow(2, retryCount)) ); } } } if (!success) { console.error( `[Image] Failed to generate image ${ promptIndex + 1 } after ${maxRetries} attempts` ); } } console.log( "[generateImagesForStory] Completed. Final segments:", localSegments.map((seg) => ({ ...seg, images: seg.images?.map((img) => (img ? "image" : "null")), })) ); return localSegments[segmentIndex]?.images || []; } catch (error) { console.error("[generateImagesForStory] Error:", error); return []; } }; // Fonction pour jouer l'audio const playAudio = async (text) => { try { // Nettoyer le texte des balises markdown et des chips const cleanText = text.replace(/\*\*(.*?)\*\*/g, "$1"); // Appeler l'API text-to-speech const response = await api.post(`${API_URL}/api/text-to-speech`, { text: cleanText, }); if (response.data.success) { // Créer un Blob à partir du base64 const audioBlob = await fetch( `data:audio/mpeg;base64,${response.data.audio_base64}` ).then((r) => r.blob()); const audioUrl = URL.createObjectURL(audioBlob); // Mettre à jour la source de l'audio audioRef.current.src = audioUrl; audioRef.current.play(); // Nettoyer l'URL quand l'audio est terminé audioRef.current.onended = () => { URL.revokeObjectURL(audioUrl); }; } } catch (error) { console.error("Error playing audio:", error); } }; const handleStoryAction = async (action, choiceId = null) => { setIsLoading(true); try { // 1. D'abord, obtenir l'histoire const response = await api.post( `${API_URL}/api/${isDebugMode ? "test/" : ""}chat`, { message: action, choice_id: choiceId, } ); // 2. Créer le nouveau segment sans images const newSegment = { text: formatTextWithBold(response.data.story_text, true), isChoice: false, isDeath: response.data.is_death, isVictory: response.data.is_victory, radiationLevel: response.data.radiation_level, is_first_step: response.data.is_first_step, is_last_step: response.data.is_last_step, images: response.data.image_prompts ? Array(response.data.image_prompts.length).fill(null) : [], // Pré-remplir avec null pour les spinners }; // 3. Calculer le nouvel index et les segments mis à jour let segmentIndex; let updatedSegments; if (action === "restart") { segmentIndex = 0; updatedSegments = [newSegment]; } else { // Récupérer l'état actuel de manière synchrone segmentIndex = storySegments.length; updatedSegments = [...storySegments, newSegment]; } // Mettre à jour l'état avec les nouveaux segments setStorySegments(updatedSegments); // 4. Mettre à jour les choix immédiatement setCurrentChoices(response.data.choices); // 5. Désactiver le loading car l'histoire est affichée setIsLoading(false); // 6. Jouer l'audio du nouveau segment await playAudio(response.data.story_text); // 7. Générer les images en parallèle if ( response.data.image_prompts && response.data.image_prompts.length > 0 ) { try { console.log( "Starting image generation with prompts:", response.data.image_prompts, "for segment", segmentIndex ); // generateImagesForStory met déjà à jour le state au fur et à mesure await generateImagesForStory( response.data.image_prompts, segmentIndex, updatedSegments ); } catch (imageError) { console.error("Error generating images:", imageError); } } } catch (error) { console.error("Error:", error); // En cas d'erreur, créer un segment d'erreur qui permet de continuer const errorSegment = { text: "Le conteur d'histoires est temporairement indisponible. Veuillez réessayer dans quelques instants...", isChoice: false, isDeath: false, isVictory: false, radiationLevel: storySegments.length > 0 ? storySegments[storySegments.length - 1].radiationLevel : 0, images: [], }; // Ajouter le segment d'erreur et permettre de réessayer if (action === "restart") { setStorySegments([errorSegment]); } else { setStorySegments((prev) => [...prev, errorSegment]); } // Donner l'option de réessayer setCurrentChoices([{ id: 1, text: "Réessayer" }]); setIsLoading(false); } }; const handleChoice = async (choiceId) => { // Si c'est l'option "Réessayer", on relance la dernière action if (currentChoices.length === 1 && currentChoices[0].text === "Réessayer") { // Supprimer le segment d'erreur setStorySegments((prev) => prev.slice(0, -1)); // Réessayer la dernière action await handleStoryAction( "choice", storySegments[storySegments.length - 2]?.choiceId || null ); return; } // Comportement normal pour les autres choix const choice = currentChoices.find((c) => c.id === choiceId); setStorySegments((prev) => [ ...prev, { text: choice.text, isChoice: true, choiceId: choiceId, // Stocker l'ID du choix pour pouvoir réessayer }, ]); // Continue the story with this choice await handleStoryAction("choice", choiceId); }; // Filter out choice segments const nonChoiceSegments = storySegments.filter( (segment) => !segment.isChoice ); const handleSaveAsImage = async () => { if (comicContainerRef.current) { try { const canvas = await html2canvas(comicContainerRef.current, { scale: 2, // Meilleure qualité backgroundColor: "#242424", // Même couleur que le fond logging: false, }); // Convertir en PNG et télécharger const image = canvas.toDataURL("image/png"); const link = document.createElement("a"); link.href = image; link.download = "my-comic-story.png"; link.click(); } catch (error) { console.error("Error saving image:", error); } } }; return ( {isLoading && ( )} {currentChoices.length > 0 ? ( {currentChoices.map((choice, index) => ( Suggestion {index + 1} ))} ) : storySegments.length > 0 && storySegments[storySegments.length - 1].is_last_step ? ( ) : null} ); } export default App;