Update app.py

app.py

@@ -8,12 +8,19 @@ pipe = DiffusionPipeline.from_pretrained("cerspense/zeroscope_v2_576w", torch_dt
 pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
 pipe.enable_model_cpu_offload()
 
-def infer(prompt):
-    #prompt = "Darth Vader is surfing on waves"
+caption = gr.load(name="spaces/fffiloni/CoCa-clone")
+
+def create_image_caption(image_init):
+    cap = caption(image_init, "Beam search", 1.2, 0.5, 5, 20, fn_index=0)
+    print("cap: " + cap)
+    return cap
+
+def infer(image_init):
+    prompt = create_image_caption(image_init)
     video_frames = pipe(prompt, num_inference_steps=40, height=320, width=576, num_frames=24).frames
     video_path = export_to_video(video_frames)
     print(video_path)
-    return video_path, gr.Group.update(visible=True)
+    return prompt, video_path, gr.Group.update(visible=True)
 
 css = """
 #col-container {max-width: 510px; margin-left: auto; margin-right: auto;}
@@ -84,19 +91,21 @@ with gr.Blocks(css=css) as demo:
     with gr.Column(elem_id="col-container"):
         gr.Markdown(
             """
-            <h1 style="text-align: center;">Zeroscope Text-to-Video</h1>
+            <h1 style="text-align: center;">Zeroscope Image-to-Video</h1>
             <p style="text-align: center;">
             A watermark-free Modelscope-based video model optimized for producing high-quality 16:9 compositions and a smooth video output. <br />
+            This demo is a variation that lets you upload an image as reference for video generation. 
             </p>
             
-            [![Duplicate this Space](https://huggingface.co/datasets/huggingface/badges/raw/main/duplicate-this-space-sm.svg#center)](https://huggingface.co/spaces/fffiloni/zeroscope?duplicate=true)
+            [![Duplicate this Space](https://huggingface.co/datasets/huggingface/badges/raw/main/duplicate-this-space-sm.svg#center)](https://huggingface.co/spaces/fffiloni/zeroscope-img-to-video?duplicate=true)
             
             """
         )
 
-        prompt_in = gr.Textbox(label="Prompt", placeholder="Darth Vader is surfing on waves", elem_id="prompt-in")
+        image_init = gr.Image(label="Image Init",type="filepath", source="upload", elem_id="image-init")
         #inference_steps = gr.Slider(label="Inference Steps", minimum=10, maximum=100, step=1, value=40, interactive=False)
         submit_btn = gr.Button("Submit")
+        coca_cap = gr.Textbox(label="Caption", placeholder="CoCa Caption will be displayed here", elem_id="coca-cap-in")
         video_result = gr.Video(label="Video Output", elem_id="video-output")
 
         with gr.Group(elem_id="share-btn-container", visible=False) as share_group:
@@ -105,8 +114,8 @@ with gr.Blocks(css=css) as demo:
             share_button = gr.Button("Share to community", elem_id="share-btn")
 
     submit_btn.click(fn=infer,
-                    inputs=[prompt_in],
-                    outputs=[video_result, share_group])
+                    inputs=[image_init],
+                    outputs=[coca_cap, video_result, share_group])
     
     share_button.click(None, [], [], _js=share_js)