Update README.md (#95)

- Update README.md (6dc5b10281c1e4dccb830be602b865178c8c1316)


Co-authored-by: Barak Weiss <[email protected]>

README.md

@@ -136,14 +136,20 @@ pipe.to("cuda")
 pipe_upsample.to("cuda")
 pipe.vae.enable_tiling()
 
+def round_to_nearest_resolution_acceptable_by_vae(height, width):
+    height = height - (height % pipe.vae_spatial_compression_ratio)
+    width = width - (width % pipe.vae_spatial_compression_ratio)
+    return height, width
+
 prompt = "The video depicts a winding mountain road covered in snow, with a single vehicle traveling along it. The road is flanked by steep, rocky cliffs and sparse vegetation. The landscape is characterized by rugged terrain and a river visible in the distance. The scene captures the solitude and beauty of a winter drive through a mountainous region."
 negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted"
-expected_height, expected_width = 704, 512
+expected_height, expected_width = 512, 704
 downscale_factor = 2 / 3
 num_frames = 121
 
 # Part 1. Generate video at smaller resolution
 downscaled_height, downscaled_width = int(expected_height * downscale_factor), int(expected_width * downscale_factor)
+downscaled_height, downscaled_width = round_to_nearest_resolution_acceptable_by_vae(downscaled_height, downscaled_width)
 latents = pipe(
     conditions=None,
     prompt=prompt,
@@ -192,7 +198,7 @@ export_to_video(video, "output.mp4", fps=24)
 import torch
 from diffusers import LTXConditionPipeline, LTXLatentUpsamplePipeline
 from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXVideoCondition
-from diffusers.utils import export_to_video, load_image
+from diffusers.utils import export_to_video, load_image, load_video
 
 pipe = LTXConditionPipeline.from_pretrained("Lightricks/LTX-Video-0.9.7-dev", torch_dtype=torch.bfloat16)
 pipe_upsample = LTXLatentUpsamplePipeline.from_pretrained("Lightricks/ltxv-spatial-upscaler-0.9.7", vae=pipe.vae, torch_dtype=torch.bfloat16)
@@ -200,13 +206,18 @@ pipe.to("cuda")
 pipe_upsample.to("cuda")
 pipe.vae.enable_tiling()
 
+def round_to_nearest_resolution_acceptable_by_vae(height, width):
+    height = height - (height % pipe.vae_spatial_compression_ratio)
+    width = width - (width % pipe.vae_spatial_compression_ratio)
+    return height, width
+
 image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/penguin.png")
-video = [image]
+video = load_video(export_to_video([image])) # compress the image using video compression as the model was trained on videos
 condition1 = LTXVideoCondition(video=video, frame_index=0)
 
-prompt = "The video depicts a winding mountain road covered in snow, with a single vehicle traveling along it. The road is flanked by steep, rocky cliffs and sparse vegetation. The landscape is characterized by rugged terrain and a river visible in the distance. The scene captures the solitude and beauty of a winter drive through a mountainous region."
+prompt = "A cute little penguin takes out a book and starts reading it"
 negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted"
-expected_height, expected_width = 832, 480
+expected_height, expected_width = 480, 832
 downscale_factor = 2 / 3
 num_frames = 96
 
@@ -254,7 +265,6 @@ video = pipe(
 video = [frame.resize((expected_width, expected_height)) for frame in video]
 
 export_to_video(video, "output.mp4", fps=24)
-
 ```
 
 ### For video-to-video: 
@@ -272,8 +282,8 @@ pipe_upsample.to("cuda")
 pipe.vae.enable_tiling()
 
 def round_to_nearest_resolution_acceptable_by_vae(height, width):
-    height = height - (height % pipe.vae_temporal_compression_ratio)
-    width = width - (width % pipe.vae_temporal_compression_ratio)
+    height = height - (height % pipe.vae_spatial_compression_ratio)
+    width = width - (width % pipe.vae_spatial_compression_ratio)
     return height, width
 
 video = load_video(
@@ -331,10 +341,8 @@ video = pipe(
 video = [frame.resize((expected_width, expected_height)) for frame in video]
 
 export_to_video(video, "output.mp4", fps=24)
-
 ```
 
-
 To learn more, check out the [official documentation](https://huggingface.co/docs/diffusers/main/en/api/pipelines/ltx_video). 
 
 Diffusers also supports directly loading from the original LTX checkpoints using the `from_single_file()` method. Check out [this section](https://huggingface.co/docs/diffusers/main/en/api/pipelines/ltx_video#loading-single-files) to learn more.