Update handler.py

handler.py

@@ -1,6 +1,6 @@
 from typing import  Dict, List, Any
 import torch
-from diffusers import DPMSolverMultistepScheduler, StableDiffusionInpaintPipeline
+from diffusers import DPMSolverMultistepScheduler, StableDiffusionInpaintPipeline, AutoPipelineForInpainting, AutoPipelineForImage2Image
 from PIL import Image
 import base64
 from io import BytesIO
@@ -15,7 +15,7 @@ if device.type != 'cuda':
 class EndpointHandler():
     def __init__(self, path=""):
         # load StableDiffusionInpaintPipeline pipeline
-        self.pipe = StableDiffusionInpaintPipeline.from_pretrained(
+        self.pipe = AutoPipelineForInpainting.from_pretrained(
             "runwayml/stable-diffusion-inpainting",
             revision="fp16",
             torch_dtype=torch.float16,
@@ -25,6 +25,12 @@ class EndpointHandler():
         # move to device        
         self.pipe = self.pipe.to(device)
 
+        self.pipe2 = AutoPipelineForInpainting.from_pretrained("stabilityai/stable-diffusion-xl-refiner-1.0", torch_dtype=torch.float16, variant="fp16", use_safetensors=True)
+        self.pipe2.to("cuda")
+
+        self.pipe3 = AutoPipelineForImage2Image.from_pipe(self.pipe2)
+        
+
 
     def __call__(self, data: Any) -> List[List[Dict[str, float]]]:
         """
@@ -43,12 +49,41 @@ class EndpointHandler():
         else:
             image = None
             mask_image = None 
+
+        self.pipe.enable_xformers_memory_efficient_attention()
         
         # run inference pipeline
         out = self.pipe(prompt=prompt, image=image, mask_image=mask_image)
+
+        image = out.images[0].resize((1024, 1024))
+
+        self.pipe2.enable_xformers_memory_efficient_attention()
+
+        image = pipe(
+            prompt=prompt,
+            image=image,
+            mask_image=mask_image,
+            guidance_scale=8.0,
+            num_inference_steps=100,
+            strength=0.2,
+            generator=generator,
+            output_type="latent",  # let's keep in latent to save some VRAM
+        ).images[0]
+        
+        pipe = AutoPipelineForImage2Image.from_pipe(pipe)
+        self.pipe3.enable_xformers_memory_efficient_attention()
+        
+        image = pipe(
+            prompt=prompt,
+            image=image,
+            guidance_scale=8.0,
+            num_inference_steps=100,
+            strength=0.2,
+            generator=generator,
+        ).images[0]
             
         # return first generate PIL image
-        return out.images[0]
+        return image
     
     # helper to decode input image
     def decode_base64_image(self, image_string):