zerogpu

app.py

@@ -21,6 +21,7 @@ import torchvision.transforms as transforms
 from loguru import logger
 from huggingface_hub import hf_hub_download
 import tempfile
+import spaces
 
 from hymm_sp.sample_inference import HunyuanVideoSampler
 from hymm_sp.data_kits.data_tools import save_videos_grid
@@ -63,7 +64,7 @@ def create_args():
     args.use_linear_quadratic_schedule = False
     args.linear_schedule_end = 0.25
     args.use_deepcache = False
-    args.cpu_offload = os.environ.get("CPU_OFFLOAD", "0") == "1"
+    args.cpu_offload = False  # Always False for ZeroGPU compatibility
     args.use_sage = True
     args.save_path = './results/'
     args.save_path_suffix = ''
@@ -117,8 +118,6 @@ def create_args():
     
     return args
 
-logger.info("Initializing Hunyuan-GameCraft model...")
-
 # Define all required model files
 required_files = [
     "gamecraft_models/mp_rank_00_model_states_distill.pt",
@@ -182,11 +181,13 @@ for file_path in text_encoder_files:
             # Continue anyway as some files might be optional
 
 logger.info("All required model files are ready")
+logger.info("Initializing Hunyuan-GameCraft model...")
 
 args = create_args()
 logger.info(f"Created args, val_disable_autocast: {hasattr(args, 'val_disable_autocast')} = {getattr(args, 'val_disable_autocast', 'NOT SET')}")
-# Load model to CPU if offloading is enabled, otherwise load to GPU
-model_device = torch.device("cpu") if args.cpu_offload else torch.device("cuda")
+
+# For ZeroGPU, always load model to CPU initially (it will be moved to GPU during inference)
+model_device = torch.device("cpu")
 logger.info(f"Loading model to device: {model_device}")
 hunyuan_video_sampler = HunyuanVideoSampler.from_pretrained(
     args.ckpt, 
@@ -197,28 +198,10 @@ logger.info(f"After from_pretrained, sampler.args has val_disable_autocast: {has
 args = hunyuan_video_sampler.args
 logger.info(f"After reassigning args, val_disable_autocast: {hasattr(args, 'val_disable_autocast')} = {getattr(args, 'val_disable_autocast', 'NOT SET')}")
 
-if args.cpu_offload:
-    from diffusers.hooks import apply_group_offloading
-    onload_device = torch.device("cuda")
-    apply_group_offloading(
-        hunyuan_video_sampler.pipeline.transformer, 
-        onload_device=onload_device, 
-        offload_type="block_level", 
-        num_blocks_per_group=1
-    )
-    logger.info("Enabled CPU offloading for transformer blocks")
-else:
-    # Ensure all model components are on GPU when not using CPU offload
-    hunyuan_video_sampler.pipeline.transformer.to('cuda')
-    hunyuan_video_sampler.vae.to('cuda')
-    if hunyuan_video_sampler.text_encoder:
-        hunyuan_video_sampler.text_encoder.model.to('cuda')
-    if hunyuan_video_sampler.text_encoder_2:
-        hunyuan_video_sampler.text_encoder_2.model.to('cuda')
-    logger.info("Model components moved to GPU")
-
-logger.info("Model loaded successfully!")
+# Don't apply CPU offloading for ZeroGPU - the model stays on CPU until needed
+logger.info("Model loaded successfully on CPU, will be moved to GPU during inference")
 
+@spaces.GPU(duration=120)
 def generate_video(
     input_image,
     prompt,
@@ -236,6 +219,16 @@ def generate_video(
         if input_image is None:
             return None, "Please upload an image first!"
         
+        # Move model components to GPU for ZeroGPU inference
+        logger.info("Moving model components to GPU...")
+        hunyuan_video_sampler.pipeline.transformer.to('cuda')
+        hunyuan_video_sampler.vae.to('cuda')
+        if hunyuan_video_sampler.text_encoder:
+            hunyuan_video_sampler.text_encoder.model.to('cuda')
+        if hunyuan_video_sampler.text_encoder_2:
+            hunyuan_video_sampler.text_encoder_2.model.to('cuda')
+        logger.info("Model components moved to GPU")
+        
         action_list = action_sequence.lower().replace(" ", "").split(",") if action_sequence else ["w"]
         speed_list = [float(s.strip()) for s in action_speeds.split(",")] if action_speeds else [0.2]
         
@@ -274,10 +267,6 @@ def generate_video(
         progress(0.2, desc="Encoding image...")
         
         with torch.autocast(device_type="cuda", dtype=torch.float16, enabled=True):
-            if args.cpu_offload:
-                hunyuan_video_sampler.vae.quant_conv.to('cuda')
-                hunyuan_video_sampler.vae.encoder.to('cuda')
-            
             hunyuan_video_sampler.pipeline.vae.enable_tiling()
             
             raw_last_latents = hunyuan_video_sampler.vae.encode(
@@ -287,9 +276,6 @@ def generate_video(
             raw_ref_latents = raw_last_latents.clone()
             
             hunyuan_video_sampler.pipeline.vae.disable_tiling()
-            if args.cpu_offload:
-                hunyuan_video_sampler.vae.quant_conv.to('cpu')
-                hunyuan_video_sampler.vae.encoder.to('cpu')
         
         ref_images = [raw_ref_image]
         last_latents = raw_last_latents
@@ -329,7 +315,7 @@ def generate_video(
                 use_linear_quadratic_schedule=args.use_linear_quadratic_schedule,
                 linear_schedule_end=args.linear_schedule_end,
                 use_deepcache=args.use_deepcache,
-                cpu_offload=args.cpu_offload,
+                cpu_offload=False,  # Always False for ZeroGPU
                 ref_images=ref_images,
                 output_dir=None,
                 return_latents=True,

demo_zerogpu.py

@@ -0,0 +1,233 @@
+# PyTorch 2.8 (temporary hack)
+import os
+os.system('pip install --upgrade --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu126 "torch<2.9" spaces')
+
+# Actual demo code
+import spaces
+import torch
+from diffusers.pipelines.wan.pipeline_wan_i2v import WanImageToVideoPipeline
+from diffusers.models.transformers.transformer_wan import WanTransformer3DModel
+from diffusers.utils.export_utils import export_to_video
+import gradio as gr
+import tempfile
+import numpy as np
+from PIL import Image
+import random
+import gc
+from optimization import optimize_pipeline_
+
+
+MODEL_ID = "Wan-AI/Wan2.2-I2V-A14B-Diffusers"
+
+LANDSCAPE_WIDTH = 832
+LANDSCAPE_HEIGHT = 480
+MAX_SEED = np.iinfo(np.int32).max
+
+FIXED_FPS = 16
+MIN_FRAMES_MODEL = 8
+MAX_FRAMES_MODEL = 81
+
+MIN_DURATION = round(MIN_FRAMES_MODEL/FIXED_FPS,1)
+MAX_DURATION = round(MAX_FRAMES_MODEL/FIXED_FPS,1)
+
+
+pipe = WanImageToVideoPipeline.from_pretrained(MODEL_ID,
+    transformer=WanTransformer3DModel.from_pretrained('cbensimon/Wan2.2-I2V-A14B-bf16-Diffusers',
+        subfolder='transformer',
+        torch_dtype=torch.bfloat16,
+        device_map='cuda',
+    ),
+    transformer_2=WanTransformer3DModel.from_pretrained('cbensimon/Wan2.2-I2V-A14B-bf16-Diffusers',
+        subfolder='transformer_2',
+        torch_dtype=torch.bfloat16,
+        device_map='cuda',
+    ),
+    torch_dtype=torch.bfloat16,
+).to('cuda')
+
+for i in range(3): 
+    gc.collect()
+    torch.cuda.synchronize() 
+    torch.cuda.empty_cache()
+
+optimize_pipeline_(pipe,
+    image=Image.new('RGB', (LANDSCAPE_WIDTH, LANDSCAPE_HEIGHT)),
+    prompt='prompt',
+    height=LANDSCAPE_HEIGHT,
+    width=LANDSCAPE_WIDTH,
+    num_frames=MAX_FRAMES_MODEL,
+)
+
+
+default_prompt_i2v = "make this image come alive, cinematic motion, smooth animation"
+default_negative_prompt = "色调艳丽, 过曝, 静态, 细节模糊不清, 字幕, 风格, 作品, 画作, 画面, 静止, 整体发灰, 最差质量, 低质量, JPEG压缩残留, 丑陋的, 残缺的, 多余的手指, 画得不好的手部, 画得不好的脸部, 畸形的, 毁容的, 形态畸形的肢体, 手指融合, 静止不动的画面, 杂乱的背景, 三条腿, 背景人很多, 倒着走"
+
+
+def resize_image(image: Image.Image) -> Image.Image:
+    if image.height > image.width:
+        transposed = image.transpose(Image.Transpose.ROTATE_90)
+        resized = resize_image_landscape(transposed)
+        return resized.transpose(Image.Transpose.ROTATE_270)
+    return resize_image_landscape(image)
+
+
+def resize_image_landscape(image: Image.Image) -> Image.Image:
+    target_aspect = LANDSCAPE_WIDTH / LANDSCAPE_HEIGHT
+    width, height = image.size
+    in_aspect = width / height
+    if in_aspect > target_aspect:
+        new_width = round(height * target_aspect)
+        left = (width - new_width) // 2
+        image = image.crop((left, 0, left + new_width, height))
+    else:
+        new_height = round(width / target_aspect)
+        top = (height - new_height) // 2
+        image = image.crop((0, top, width, top + new_height))
+    return image.resize((LANDSCAPE_WIDTH, LANDSCAPE_HEIGHT), Image.LANCZOS)
+
+def get_duration(
+    input_image,
+    prompt,
+    steps,
+    negative_prompt,
+    duration_seconds,
+    guidance_scale,
+    guidance_scale_2,
+    seed,
+    randomize_seed,
+    progress,
+):
+    return int(steps) * 15
+
+@spaces.GPU(duration=get_duration)
+def generate_video(
+    input_image,
+    prompt,
+    steps = 4,
+    negative_prompt=default_negative_prompt,
+    duration_seconds = MAX_DURATION,
+    guidance_scale = 1,
+    guidance_scale_2 = 1,    
+    seed = 42,
+    randomize_seed = False,
+    progress=gr.Progress(track_tqdm=True),
+):
+    """
+    Generate a video from an input image using the Wan 2.2 14B I2V model with Lightning LoRA.
+    
+    This function takes an input image and generates a video animation based on the provided
+    prompt and parameters. It uses an FP8 qunatized Wan 2.2 14B Image-to-Video model in with Lightning LoRA
+    for fast generation in 4-8 steps.
+    
+    Args:
+        input_image (PIL.Image): The input image to animate. Will be resized to target dimensions.
+        prompt (str): Text prompt describing the desired animation or motion.
+        steps (int, optional): Number of inference steps. More steps = higher quality but slower.
+            Defaults to 4. Range: 1-30.
+        negative_prompt (str, optional): Negative prompt to avoid unwanted elements. 
+            Defaults to default_negative_prompt (contains unwanted visual artifacts).
+        duration_seconds (float, optional): Duration of the generated video in seconds.
+            Defaults to 2. Clamped between MIN_FRAMES_MODEL/FIXED_FPS and MAX_FRAMES_MODEL/FIXED_FPS.
+        guidance_scale (float, optional): Controls adherence to the prompt. Higher values = more adherence.
+            Defaults to 1.0. Range: 0.0-20.0.
+        guidance_scale_2 (float, optional): Controls adherence to the prompt. Higher values = more adherence.
+            Defaults to 1.0. Range: 0.0-20.0.
+        seed (int, optional): Random seed for reproducible results. Defaults to 42.
+            Range: 0 to MAX_SEED (2147483647).
+        randomize_seed (bool, optional): Whether to use a random seed instead of the provided seed.
+            Defaults to False.
+        progress (gr.Progress, optional): Gradio progress tracker. Defaults to gr.Progress(track_tqdm=True).
+    
+    Returns:
+        tuple: A tuple containing:
+            - video_path (str): Path to the generated video file (.mp4)
+            - current_seed (int): The seed used for generation (useful when randomize_seed=True)
+    
+    Raises:
+        gr.Error: If input_image is None (no image uploaded).
+    
+    Note:
+        - The function automatically resizes the input image to the target dimensions
+        - Frame count is calculated as duration_seconds * FIXED_FPS (24)
+        - Output dimensions are adjusted to be multiples of MOD_VALUE (32)
+        - The function uses GPU acceleration via the @spaces.GPU decorator
+        - Generation time varies based on steps and duration (see get_duration function)
+    """
+    if input_image is None:
+        raise gr.Error("Please upload an input image.")
+    
+    num_frames = np.clip(int(round(duration_seconds * FIXED_FPS)), MIN_FRAMES_MODEL, MAX_FRAMES_MODEL)
+    current_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed)
+    resized_image = resize_image(input_image)
+
+    output_frames_list = pipe(
+        image=resized_image,
+        prompt=prompt,
+        negative_prompt=negative_prompt,
+        height=resized_image.height,
+        width=resized_image.width,
+        num_frames=num_frames,
+        guidance_scale=float(guidance_scale),
+        guidance_scale_2=float(guidance_scale_2),
+        num_inference_steps=int(steps),
+        generator=torch.Generator(device="cuda").manual_seed(current_seed),
+    ).frames[0]
+
+    with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tmpfile:
+        video_path = tmpfile.name
+
+    export_to_video(output_frames_list, video_path, fps=FIXED_FPS)
+
+    return video_path, current_seed
+
+with gr.Blocks() as demo:
+    gr.Markdown("# Fast 4 steps Wan 2.2 I2V (14B) with Lightning LoRA")
+    gr.Markdown("run Wan 2.2 in just 4-8 steps, with [Lightning LoRA](https://huggingface.co/Kijai/WanVideo_comfy/tree/main/Wan22-Lightning), fp8 quantization & AoT compilation - compatible with 🧨 diffusers and ZeroGPU⚡️")
+    with gr.Row():
+        with gr.Column():
+            input_image_component = gr.Image(type="pil", label="Input Image (auto-resized to target H/W)")
+            prompt_input = gr.Textbox(label="Prompt", value=default_prompt_i2v)
+            duration_seconds_input = gr.Slider(minimum=MIN_DURATION, maximum=MAX_DURATION, step=0.1, value=3.5, label="Duration (seconds)", info=f"Clamped to model's {MIN_FRAMES_MODEL}-{MAX_FRAMES_MODEL} frames at {FIXED_FPS}fps.")
+            
+            with gr.Accordion("Advanced Settings", open=False):
+                negative_prompt_input = gr.Textbox(label="Negative Prompt", value=default_negative_prompt, lines=3)
+                seed_input = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=42, interactive=True)
+                randomize_seed_checkbox = gr.Checkbox(label="Randomize seed", value=True, interactive=True)
+                steps_slider = gr.Slider(minimum=1, maximum=30, step=1, value=6, label="Inference Steps") 
+                guidance_scale_input = gr.Slider(minimum=0.0, maximum=10.0, step=0.5, value=1, label="Guidance Scale - high noise stage")
+                guidance_scale_2_input = gr.Slider(minimum=0.0, maximum=10.0, step=0.5, value=1, label="Guidance Scale 2 - low noise stage")
+
+            generate_button = gr.Button("Generate Video", variant="primary")
+        with gr.Column():
+            video_output = gr.Video(label="Generated Video", autoplay=True, interactive=False)
+    
+    ui_inputs = [
+        input_image_component, prompt_input, steps_slider,
+        negative_prompt_input, duration_seconds_input,
+        guidance_scale_input, guidance_scale_2_input, seed_input, randomize_seed_checkbox
+    ]
+    generate_button.click(fn=generate_video, inputs=ui_inputs, outputs=[video_output, seed_input])
+
+    gr.Examples(
+        examples=[ 
+            [
+                "wan_i2v_input.JPG",
+                "POV selfie video, white cat with sunglasses standing on surfboard, relaxed smile, tropical beach behind (clear water, green hills, blue sky with clouds). Surfboard tips, cat falls into ocean, camera plunges underwater with bubbles and sunlight beams. Brief underwater view of cat’s face, then cat resurfaces, still filming selfie, playful summer vacation mood.",
+                4,
+            ],
+            [
+                "wan22_input_2.jpg",
+                "A sleek lunar vehicle glides into view from left to right, kicking up moon dust as astronauts in white spacesuits hop aboard with characteristic lunar bouncing movements. In the distant background, a VTOL craft descends straight down and lands silently on the surface. Throughout the entire scene, ethereal aurora borealis ribbons dance across the star-filled sky, casting shimmering curtains of green, blue, and purple light that bathe the lunar landscape in an otherworldly, magical glow.",
+                4,
+            ],
+            [
+                "kill_bill.jpeg",
+                "Uma Thurman's character, Beatrix Kiddo, holds her razor-sharp katana blade steady in the cinematic lighting. Suddenly, the polished steel begins to soften and distort, like heated metal starting to lose its structural integrity. The blade's perfect edge slowly warps and droops, molten steel beginning to flow downward in silvery rivulets while maintaining its metallic sheen. The transformation starts subtly at first - a slight bend in the blade - then accelerates as the metal becomes increasingly fluid. The camera holds steady on her face as her piercing eyes gradually narrow, not with lethal focus, but with confusion and growing alarm as she watches her weapon dissolve before her eyes. Her breathing quickens slightly as she witnesses this impossible transformation. The melting intensifies, the katana's perfect form becoming increasingly abstract, dripping like liquid mercury from her grip. Molten droplets fall to the ground with soft metallic impacts. Her expression shifts from calm readiness to bewilderment and concern as her legendary instrument of vengeance literally liquefies in her hands, leaving her defenseless and disoriented.",
+                6,
+            ],
+        ],
+        inputs=[input_image_component, prompt_input, steps_slider], outputs=[video_output, seed_input], fn=generate_video, cache_examples="lazy"
+    )
+
+if __name__ == "__main__":
+    demo.queue().launch(mcp_server=True)

docs_for_ai_coding_bots/spaces/zero-gpu.md

@@ -0,0 +1,103 @@
+[](#spaces-zerogpu-dynamic-gpu-allocation-for-spaces)Spaces ZeroGPU: Dynamic GPU Allocation for Spaces
+======================================================================================================
+
+![ZeroGPU schema](https://cdn-uploads.huggingface.co/production/uploads/5f17f0a0925b9863e28ad517/naVZI-v41zNxmGlhEhGDJ.gif)
+
+ZeroGPU is a shared infrastructure that optimizes GPU usage for AI models and demos on Hugging Face Spaces. It dynamically allocates and releases NVIDIA H200 GPUs as needed, offering:
+
+1.  **Free GPU Access**: Enables cost-effective GPU usage for Spaces.
+2.  **Multi-GPU Support**: Allows Spaces to leverage multiple GPUs concurrently on a single application.
+
+Unlike traditional single-GPU allocations, ZeroGPU’s efficient system lowers barriers for developers, researchers, and organizations to deploy AI models by maximizing resource utilization and power efficiency.
+
+[](#using-and-hosting-zerogpu-spaces)Using and hosting ZeroGPU Spaces
+---------------------------------------------------------------------
+
+*   **Using existing ZeroGPU Spaces**
+    *   ZeroGPU Spaces are available to use for free to all users. (Visit [the curated list](https://huggingface.co/spaces/enzostvs/zero-gpu-spaces)).
+    *   [PRO users](https://huggingface.co/subscribe/pro) get x5 more daily usage quota and highest priority in GPU queues when using any ZeroGPU Spaces.
+*   **Hosting your own ZeroGPU Spaces**
+    *   Personal accounts: [Subscribe to PRO](https://huggingface.co/settings/billing/subscription) to access ZeroGPU in the hardware options when creating a new Gradio SDK Space.
+    *   Organizations: [Subscribe to the Enterprise Hub](https://huggingface.co/enterprise) to enable ZeroGPU Spaces for all organization members.
+
+[](#technical-specifications)Technical Specifications
+-----------------------------------------------------
+
+*   **GPU Type**: Nvidia H200 slice
+*   **Available VRAM**: 70GB per workload
+
+[](#compatibility)Compatibility
+-------------------------------
+
+ZeroGPU Spaces are designed to be compatible with most PyTorch-based GPU Spaces. While compatibility is enhanced for high-level Hugging Face libraries like `transformers` and `diffusers`, users should be aware that:
+
+*   Currently, ZeroGPU Spaces are exclusively compatible with the **Gradio SDK**.
+*   ZeroGPU Spaces may have limited compatibility compared to standard GPU Spaces.
+*   Unexpected issues may arise in some scenarios.
+
+### [](#supported-versions)Supported Versions
+
+*   Gradio: 4+
+*   PyTorch: 2.1.2, 2.2.2, 2.4.0, 2.5.1 (Note: 2.3.x is not supported due to a [PyTorch bug](https://github.com/pytorch/pytorch/issues/122085))
+*   Python: 3.10.13
+
+[](#getting-started-with-zerogpu)Getting started with ZeroGPU
+-------------------------------------------------------------
+
+To utilize ZeroGPU in your Space, follow these steps:
+
+1.  Make sure the ZeroGPU hardware is selected in your Space settings.
+2.  Import the `spaces` module.
+3.  Decorate GPU-dependent functions with `@spaces.GPU`.
+
+This decoration process allows the Space to request a GPU when the function is called and release it upon completion.
+
+### [](#example-usage)Example Usage
+
+Copied
+
+import spaces
+from diffusers import DiffusionPipeline
+
+pipe = DiffusionPipeline.from\_pretrained(...)
+pipe.to('cuda')
+
+@spaces.GPU
+def generate(prompt):
+    return pipe(prompt).images
+
+gr.Interface(
+    fn=generate,
+    inputs=gr.Text(),
+    outputs=gr.Gallery(),
+).launch()
+
+Note: The `@spaces.GPU` decorator is designed to be effect-free in non-ZeroGPU environments, ensuring compatibility across different setups.
+
+[](#duration-management)Duration Management
+-------------------------------------------
+
+For functions expected to exceed the default 60-second of GPU runtime, you can specify a custom duration:
+
+Copied
+
+@spaces.GPU(duration=120)
+def generate(prompt):
+   return pipe(prompt).images
+
+This sets the maximum function runtime to 120 seconds. Specifying shorter durations for quicker functions will improve queue priority for Space visitors.
+
+[](#hosting-limitations)Hosting Limitations
+-------------------------------------------
+
+*   **Personal accounts ([PRO subscribers](https://huggingface.co/subscribe/pro))**: Maximum of 10 ZeroGPU Spaces.
+*   **Organization accounts ([Enterprise Hub](https://huggingface.co/enterprise))**: Maximum of 50 ZeroGPU Spaces.
+
+By leveraging ZeroGPU, developers can create more efficient and scalable Spaces, maximizing GPU utilization while minimizing costs.
+
+[](#feedback)Feedback
+---------------------
+
+You can share your feedback on Spaces ZeroGPU directly on the HF Hub: [https://huggingface.co/spaces/zero-gpu-explorers/README/discussions](https://huggingface.co/spaces/zero-gpu-explorers/README/discussions)
+
+[< \> Update on GitHub](https://github.com/huggingface/hub-docs/blob/main/docs/hub/spaces-zerogpu.md)

requirements.txt

@@ -85,3 +85,4 @@ wheel==0.45.1
 zipp==3.23.0
 gradio==5.42.0
 sageattention==1.0.6
+spaces