merge CLIPSeg demo

app.py

@@ -1,12 +1,17 @@
-import gradio as gr
-
-from PIL import Image
 import os
 import torch
+import gradio as gr
+from PIL import Image
+import matplotlib.pyplot as plt
 from diffusers import DiffusionPipeline
+from transformers import CLIPSegProcessor, CLIPSegForImageSegmentation
 
 from share_btn import community_icon_html, loading_icon_html, share_js
 
+
+processor = CLIPSegProcessor.from_pretrained("CIDAS/clipseg-rd64-refined")
+model = CLIPSegForImageSegmentation.from_pretrained("CIDAS/clipseg-rd64-refined")
+
 pipe = DiffusionPipeline.from_pretrained(
     "Fantasy-Studio/Paint-by-Example",
     torch_dtype=torch.float16,
@@ -14,14 +19,25 @@ pipe = DiffusionPipeline.from_pretrained(
 pipe = pipe.to("cuda")
 
 
-def read_content(file_path: str) -> str:
-    """read the content of target file
-    """
+def process_image(image, prompt):
+    inputs = processor(text=prompt, images=image, padding="max_length", return_tensors="pt")
+  
+    # predict
+    with torch.no_grad():
+        outputs = model(**inputs)
+        preds = outputs.logits
+  
+    filename = f"mask.png"
+    plt.imsave(filename, torch.sigmoid(preds))
+    return Image.open("mask.png").convert("RGB")
+
+
+def read_content(file_path):
     with open(file_path, 'r', encoding='utf-8') as f:
         content = f.read()
-
     return content
 
+
 def predict(dict, reference, scale, seed, step):
     width, height = dict["image"].size
     if width < height:
@@ -123,8 +139,7 @@ with image_blocks as demo:
                         community_icon = gr.HTML(community_icon_html, visible=True)
                         loading_icon = gr.HTML(loading_icon_html, visible=True)
                         share_button = gr.Button("Share to community", elem_id="share-btn", visible=True)
-            
-            
+
             with gr.Row():
                 with gr.Column():
                     gr.Examples(image_list, inputs=[image],label="Examples - Source Image",examples_per_page=12)
@@ -134,8 +149,6 @@ with image_blocks as demo:
             btn.click(fn=predict, inputs=[image, reference, guidance, seed, steps], outputs=[image_out, community_icon, loading_icon, share_button])
             share_button.click(None, [], [], _js=share_js)
 
-
-
             gr.HTML(
                 """
                     <div class="footer">