reformulate demo interface

README.md

@@ -4,7 +4,7 @@ emoji: 📚
 colorFrom: yellow
 colorTo: gray
 sdk: gradio
-sdk_version: 5.1.0
+sdk_version: 5.4.0
 app_file: app.py
 pinned: false
 license: mit

app.py

@@ -5,6 +5,9 @@ import os
 import time
 import hashlib
 import uuid
+import traceback
+import logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
 
 import spaces
 import gradio as gr
@@ -51,11 +54,11 @@ asr_format = "Please recognize the text corresponding to the follwing speech.\n"
 tts_format = "Please synthesize the speech corresponding to the follwing text.\n"
 chat_format = r'Please recognize the texts, emotion and pitch from the user question speech units and provide the texts, emotion, pitch and speech units for the assistant response. \nEmotion should be chosen from ["neutral", "happy", "sad", "angry", "surprised", "disgusted", "fearful"]. \nPitch should be chosen from ["low", "normal", "high"].\nYour output should be in json format.\nAn output example is:\n{"user question text": "", "user question emotion": "", "user question pitch": "", "assistant response text": "", "assistant response emotion": "", "assistant response pitch": ""，"assistant response speech": ""}\n\nuser question speech:'
 
-@spaces.GPU(duration=20)
+@spaces.GPU(duration=15)
 def s2u_asr(text, audio_file):
     return asr_format + s2u_extract_unit_demo(s2u_model, audio_file, model_name=s2u_model_name, reduced=reduced)
 
-@spaces.GPU(duration=20)
+@spaces.GPU(duration=15)
 def s2u_chat(text, audio_file):
     return chat_format + s2u_extract_unit_demo(s2u_model, audio_file, model_name=s2u_model_name, reduced=reduced)
 
@@ -108,12 +111,12 @@ disable_btn = gr.Button(interactive=False)
 server_error_msg = "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**"
 
 def load_demo_refresh_model_list():
-    print(f"load_demo.")
+    logging.info(f"load_demo.")
     state = default_conversation.copy()
     return state
 
 def regenerate(state, image_process_mode):
-    print(f"regenerate.")
+    logging.info(f"regenerate.")
     state.messages[-1][-1] = None
     prev_human_msg = state.messages[-2]
     if type(prev_human_msg[1]) in (tuple, list):
@@ -122,7 +125,7 @@ def regenerate(state, image_process_mode):
     return (state, state.to_gradio_chatbot_public(), "", None, None) + (disable_btn,) * 2
 
 def clear_history():
-    print(f"clear_history.")
+    logging.info(f"clear_history.")
     state = default_conversation.copy()
     return (state, state.to_gradio_chatbot_public(), "", None) + (disable_btn,) * 2 + (None,)
 
@@ -135,7 +138,7 @@ def add_text(state, text, image, image_process_mode, audio_input, audio_mode):
     ############
     # Input legality checking
     ############
-    print(f"add_text. len: {len(text)}")
+    logging.info(f"add_text. len: {len(text)}")
     if len(text) <= 0 and image is None and audio_input is None:
         state.skip_next = True
         return (state, state.to_gradio_chatbot_public(), "", None, None) + (no_change_btn,) * 2
@@ -178,7 +181,7 @@ def add_text(state, text, image, image_process_mode, audio_input, audio_mode):
     state.append_message(state.roles[0], text)
     state.append_message(state.roles[1], None)
     state.skip_next = False
-    print(str(state.messages))
+    logging.info(str(state.messages))
     return (state, state.to_gradio_chatbot_public(), "", None, None) + (disable_btn,) * 2
 
 ############
@@ -186,9 +189,9 @@ def add_text(state, text, image, image_process_mode, audio_input, audio_mode):
 # Input: [state, temperature, top_p, max_output_tokens, speaker]
 # Return: [state, chatbot] + btn_list
 ############
-@spaces.GPU(duration=120)
+@spaces.GPU(duration=90)
 def http_bot(state, temperature, top_p, max_new_tokens, speaker):
-    print(f"http_bot.")
+    logging.info(f"http_bot.")
 
     if state.skip_next:
         yield (state, state.to_gradio_chatbot_public()) + (no_change_btn,) * 2
@@ -196,35 +199,12 @@ def http_bot(state, temperature, top_p, max_new_tokens, speaker):
 
     if len(state.messages) == state.offset + 2:
         # First round of conversation
-        if 'llama-2' in model_name.lower():
-            template_name = "llava_llama_2"
-        elif "mistral" in model_name.lower() or "mixtral" in model_name.lower():
-            if 'orca' in model_name.lower():
-                template_name = "mistral_orca"
-            elif 'hermes' in model_name.lower():
-                template_name = "chatml_direct"
-            else:
-                template_name = "mistral_instruct"
-        elif 'llava-v1.6-34b' in model_name.lower():
-            template_name = "chatml_direct"
-        elif "v1" in model_name.lower():
-            if 'mmtag' in model_name.lower():
-                template_name = "v1_mmtag"
-            elif 'plain' in model_name.lower() and 'finetune' not in model_name.lower():
-                template_name = "v1_mmtag"
-            else:
-                template_name = "llava_v1"
-        elif "mpt" in model_name.lower():
-            template_name = "mpt"
-        elif "llama3" in model_name.lower():
+        if "llama3" in model_name.lower():
             template_name = 'llama3_demo'
+        elif "qwen2" in model_name.lower():
+            template_name = 'qwen2_demo'
         else:
-            if 'mmtag' in model_name.lower():
-                template_name = "v0_mmtag"
-            elif 'plain' in model_name.lower() and 'finetune' not in model_name.lower():
-                template_name = "v0_mmtag"
-            else:
-                template_name = "llava_v0"
+            template_name = "default"
 
         new_state = conv_templates[template_name].copy()
         new_state.append_message(new_state.roles[0], state.messages[-2][1])
@@ -234,7 +214,6 @@ def http_bot(state, temperature, top_p, max_new_tokens, speaker):
     # Construct prompt
     prompt = state.get_prompt()
     all_images = state.get_images(return_pil=True)
-    all_image_hash = [hashlib.md5(image.tobytes()).hexdigest() for image in all_images]
     
     # Make requests
     pload = {
@@ -244,10 +223,9 @@ def http_bot(state, temperature, top_p, max_new_tokens, speaker):
         "top_p": float(top_p),
         "max_new_tokens": int(max_new_tokens),
         "stop": state.sep if state.sep_style in [SeparatorStyle.SINGLE, SeparatorStyle.MPT] else state.sep2,
-        "images": f'List of {len(state.get_images())} images: {all_image_hash}',
+        "images": f'List of {len(state.get_images())} images: {all_images}',
     }
-    print(f"==== request ====\n{pload}")
-    pload['images'] = all_images
+    logging.info(f"==== request ====\n{pload}")
     
     # Process inputs
     inputs = processor(text=[prompt], images=all_images if len(all_images) > 0 else None, return_tensors="pt")
@@ -290,7 +268,8 @@ def http_bot(state, temperature, top_p, max_new_tokens, speaker):
                 # state.messages[-1][-1] = "[😁 GENERATING AUDIO {}%...]".format(round(output.count("<|speech_") / max_new_tokens * 100, 1)) + "\n" + output + "▌"
             yield (state, state.to_gradio_chatbot_public()) + (disable_btn,) * 2
     except Exception as e:
-        print(e)
+        os.system("nvidia-smi")
+        logging.info(traceback.print_exc())
         state.messages[-1][-1] = server_error_msg
         yield (state, state.to_gradio_chatbot_public()) + (enable_btn,) * 2
         return
@@ -323,14 +302,15 @@ def http_bot(state, temperature, top_p, max_new_tokens, speaker):
             condition = f'gender-{gender}_emotion-{emotion}_speed-{speed}_pitch-{pitch}'
             style_centroid_file = condition2style_centroid_file_dict[condition]
             style_centroid_embedding = condition2style_centroid_embedding_dict[condition].cuda()
-            print(condition)
+            logging.info(condition)
         
             id = str(uuid.uuid4())
             os.makedirs("./demo_audio", exist_ok=True)    
             synthesis(content_unit, style_centroid_embedding, hps, net_g, f"./demo_audio/{id}_temp_audio.wav")
             temp_file = f"./demo_audio/{id}_temp_audio.wav"
         except Exception as e:
-            print(e)
+            os.system("nvidia-smi")
+            logging.info(traceback.print_exc())
 
     state.messages[-1][-1] = state.messages[-1][-1][:-1]
     if tts_format in prompt or chat_format in prompt:
@@ -346,39 +326,48 @@ def http_bot(state, temperature, top_p, max_new_tokens, speaker):
     if temp_file is not None:
         os.system("rm {}".format(temp_file))
 
-    print(f"{output}")
+    logging.info(f"{output}")
 
 ############
 # Layout Markdown
 ############
 title_markdown = ("""
 <div style="display: flex; align-items: center; padding: 20px; border-radius: 10px; background-color: #f0f0f0;">
-  <div style="margin-right: 20px;">
+  <div style="margin-left: 20px; margin-right: 40px;">
     <img src="https://emova-ollm.github.io/static/images/icons/emova.png" alt="Icon" style="width: 100px; height: 100px; border-radius: 10px;">
   </div>
   <div>
-    <h1 style="margin: 0;">EMOVA: Empowering Language Models to See, Hear and Speak with Vivid Emotion</h2>
-    <p style="margin: 10px 0;">
-      1. To chat with EMOVA, upload images, enter texts or record audios and then do not forget to <mark>Click 💬 Chat Button</mark> ^v^!<br/>
-      2. Heighten the <code>Max output tokens</code> if necessary to talk longer with EMOVA.
+    <h1 style="margin: 0;">EMOVA: Empowering Language Models to See, Hear and Speak with Vivid Emotions</h1>
+    <h2 style="margin: 10px 0;">📃 <a href="https://arxiv.org/abs/2409.18042" style="font-weight: 300;">Paper</a> | 💻 <a href="https://github.com/emova-ollm/EMOVA" style="font-weight: 300;">Code</a> | 🤗 <a href="https://huggingface.co/Emova-ollm" style="font-weight: 300;">HuggingFace</a> | 🌐 <a href="https://emova-ollm.github.io/" style="font-weight: 300;">Website</a></h2>
+    <p  style="margin: 20px 0;">
+      <strong>1. To chat with EMOVA, upload images, enter texts or record audios and then do not forget to <mark>Click 💬 Chat Button</mark> ^v^!</strong><br/>
+      <strong>2. Heighten the <code>Max output tokens</code> if necessary to talk longer with EMOVA.</strong>
     </p>
   </div>
 </div>
 """)
 
 tos_markdown = ("""
-### Terms of use
+## Terms of use
 By using this service, users are required to agree to the following terms:
 The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. The service may collect user dialogue data for future research.
 For an optimal experience, please use desktop computers for this demo, as mobile devices may compromise its quality.
 """)
 
 learn_more_markdown = ("""
-### License
+## License
 The service is a research preview intended for non-commercial use only, subject to the model [License](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) of LLaMA and [Privacy Practices](https://chrome.google.com/webstore/detail/sharegpt-share-your-chatg/daiacboceoaocpibfodeljbdfacokfjb) of ShareGPT. Please contact us if you find any potential violation.
 
-### Acknowledgement
+## Acknowledgement
 The service is built upon [LLaVA](https://github.com/haotian-liu/LLaVA/). We thanks the authors for open-sourcing the wonderful code.
+
+## Citation
+<pre><code>@article{chen2024emova,
+  title={Emova: Empowering language models to see, hear and speak with vivid emotions},
+  author={Chen, Kai and Gou, Yunhao and Huang, Runhui and Liu, Zhili and Tan, Daxin and Xu, Jing and Wang, Chunwei and Zhu, Yi and Zeng, Yihan and Yang, Kuo and others},
+  journal={arXiv preprint arXiv:2409.18042},
+  year={2024}
+}</code></pre>
 """)
 
 block_css = """
@@ -398,7 +387,7 @@ block_css = """
 ############
 # Layout Demo
 ############
-def build_demo(embed_mode, cur_dir=None):
+def build_demo(embed_mode):
     textbox = gr.Textbox(label="Text", show_label=False, placeholder="Enter text or record audio in the right and then click 💬 Chat to talk with me ^v^", container=False, scale=6)
     audio_input = gr.Audio(label="Audio", sources=["microphone", "upload"], type="filepath", max_length=10, show_download_button=True, waveform_options=dict(sample_rate=16000), scale=2)
     with gr.Blocks(title="EMOVA", theme=gr.themes.Default(), css=block_css) as demo:
@@ -424,7 +413,7 @@ def build_demo(embed_mode, cur_dir=None):
                     temperature = gr.Slider(minimum=0.0, maximum=1.0, value=0.2, step=0.1, interactive=True, label="Temperature")
                     top_p = gr.Slider(minimum=0.0, maximum=1.0, value=0.7, step=0.1, interactive=True, label="Top P")
                     max_output_tokens = gr.Slider(minimum=0, maximum=4096, value=2048, step=32, interactive=True, label="Max output tokens")
-                    speaker = gr.Radio(["Female", "Male"], label="Speaker")
+                    speaker = gr.Radio(["Female", "Male"], value="Female", label="Speaker")
 
             with gr.Column(scale=8):
                 chatbot = gr.Chatbot(
@@ -445,15 +434,12 @@ def build_demo(embed_mode, cur_dir=None):
         ##############
         # Examples
         ##############
-        if cur_dir is None:
-            cur_dir = os.path.dirname(os.path.abspath(__file__))
-
         with gr.Row():
             with gr.Column(scale=9):
                 gr.Examples(examples=[
                     ["./examples/emo-speech/what_is_your_name.wav"],
-                    ["./examples/emo-speech/parent.wav"],
                     ["./examples/emo-speech/I_am_so_sad.wav"],
+                    ["./examples/emo-speech/parent.wav"],
                     ["./examples/emo-speech/wedding(CH).wav"],
                 ], inputs=[audio_input], label='Audio Examples (Click to load the examples~)')
 
@@ -538,6 +524,7 @@ args = parser.parse_args()
 
 demo = build_demo(args.embed)
 demo.queue(
+    max_size=10,
     api_open=False
 ).launch(
     favicon_path="./examples/icon_256.png",

conversation_public.py

@@ -462,6 +462,16 @@ conv_qwen2 = Conversation(
     sep="<|im_end|>\n",
 )
 
+conv_qwen2_demo = Conversation(
+    system='<|im_start|>system\nYou are a helpful assistant. Your name is emova, and you are purely developed by the emova Team.',
+    roles=("<|im_start|>user\n", "<|im_start|>assistant\n"),
+    version="qwen2_demo",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.MPT,
+    sep="<|im_end|>\n",
+)
+
 conv_glm4 = Conversation(
     system='[gMASK]<sop>',
     roles=("<|user|>", "<|assistant|>"),
@@ -498,6 +508,7 @@ conv_templates = {
 
     "mpt": conv_mpt,
     "qwen2": conv_qwen2,
+    "qwen2_demo": conv_qwen2_demo,
     "glm4": conv_glm4,
 }
 

requirements.txt

@@ -1,7 +1,7 @@
 omegaconf
 # torch
 # torchvision
-transformers==4.44.0
+transformers==4.44.2
 sentencepiece==0.1.99
 accelerate==0.33.0
 einops==0.6.1
@@ -10,7 +10,7 @@ timm==0.6.13
 # flash_attn
 https://github.com/Dao-AILab/flash-attention/releases/download/v2.6.3/flash_attn-2.6.3+cu123torch2.4cxx11abiFALSE-cp310-cp310-linux_x86_64.whl
 scipy
-gradio
+# gradio
 
 monotonic_align
 librosa==0.8.0