Update optimization.py

optimization.py

@@ -1,31 +1,14 @@
-import torch
-import torchao
-from torchao.quantization import DEFAULT_INT4_AUTOQUANT_CLASS_LIST
-from typing import Any
-from typing import Callable
-from typing import ParamSpec
-
+from typing import Any, Callable, ParamSpec
 import spaces
 import torch
 from torch.utils._pytree import tree_map_only
-from torchao.quantization import quantize_
-from torchao.quantization import Float8DynamicActivationFloat8WeightConfig
-from torchao.quantization import Int8WeightOnlyConfig
-
-from optimization_utils import capture_component_call
-from optimization_utils import aoti_compile
-from optimization_utils import ZeroGPUCompiledModel
-from optimization_utils import drain_module_parameters
-P = ParamSpec('P')
+from torchao.quantization import quantize_, Float8DynamicActivationFloat8WeightConfig, Int8WeightOnlyConfig
+from optimization_utils import capture_component_call, aoti_compile, ZeroGPUCompiledModel, drain_module_parameters
 
+P = ParamSpec('P')
 
 TRANSFORMER_NUM_FRAMES_DIM = torch.export.Dim('num_frames', min=3, max=21)
-
-TRANSFORMER_DYNAMIC_SHAPES = {
-    'hidden_states': {
-        2: TRANSFORMER_NUM_FRAMES_DIM,
-    },
-}
+TRANSFORMER_DYNAMIC_SHAPES = {'hidden_states': {2: TRANSFORMER_NUM_FRAMES_DIM}}
 
 INDUCTOR_CONFIGS = {
     'conv_1x1_as_mm': True,
@@ -36,22 +19,14 @@ INDUCTOR_CONFIGS = {
     'triton.cudagraphs': True,
 }
 
-from torchao.quantization import DEFAULT_INT4_AUTOQUANT_CLASS_LIST
-
 def optimize_pipeline_(pipeline: Callable[P, Any], *args: P.args, **kwargs: P.kwargs):
     print("[optimize_pipeline_] Starting pipeline optimization")
 
-    # --- TEXT ENCODER ---
-    pipeline.text_encoder = pipeline.text_encoder.cpu()
-    pipeline.text_encoder = torchao.autoquant(
-        torch.compile(pipeline.text_encoder, mode='max-autotune'),
-        qtensor_class_list=DEFAULT_INT4_AUTOQUANT_CLASS_LIST
-    ).to("cuda")
-    print("[optimize_pipeline_] Text encoder autoquantized and compiled")
+    quantize_(pipeline.text_encoder, Int8WeightOnlyConfig())
+    print("[optimize_pipeline_] Text encoder quantized")
 
     @spaces.GPU(duration=1500)
     def compile_transformer():
-        # --- LOAD LORAS ---
         print("[compile_transformer] Loading LoRA weights")
         pipeline.load_lora_weights(
             "DeepBeepMeep/Wan2.2",
@@ -66,64 +41,80 @@ def optimize_pipeline_(pipeline: Callable[P, Any], *args: P.args, **kwargs: P.kw
         )
         pipeline.set_adapters(["lightning", "lightning_2"], adapter_weights=[1.0, 1.0])
 
-        # --- FUSE & UNLOAD ---
         print("[compile_transformer] Fusing LoRA weights")
         pipeline.fuse_lora(adapter_names=["lightning"], lora_scale=3.0, components=["transformer"])
         pipeline.fuse_lora(adapter_names=["lightning_2"], lora_scale=1.0, components=["transformer_2"])
         pipeline.unload_lora_weights()
 
-        # --- DUMMY FORWARD ---
-        print("[compile_transformer] Capturing shapes")
+        print("[compile_transformer] Running dummy forward pass to capture component call")
         with torch.inference_mode():
             with capture_component_call(pipeline, 'transformer') as call:
                 pipeline(*args, **kwargs)
 
-        hidden_states: torch.Tensor = call.kwargs['hidden_states']
-        hidden_states_transposed = hidden_states.transpose(-1, -2).contiguous()
+        dynamic_shapes = tree_map_only((torch.Tensor, bool), lambda t: None, call.kwargs)
+        dynamic_shapes |= TRANSFORMER_DYNAMIC_SHAPES
 
-        # --- MOVE TO CUDA BEFORE AUTOQUANT ---
-        pipeline.transformer = pipeline.transformer.to("cuda")
-        pipeline.transformer_2 = pipeline.transformer_2.to("cuda")
+        print("[compile_transformer] Quantizing transformers with Float8DynamicActivationFloat8WeightConfig")
+        quantize_(pipeline.transformer, Float8DynamicActivationFloat8WeightConfig())
+        quantize_(pipeline.transformer_2, Float8DynamicActivationFloat8WeightConfig())
 
-        # Sanity: Ensure parameters exist before quantization
-        assert any(p.numel() > 0 for p in pipeline.transformer.parameters()), "Transformer has no params!"
-        assert any(p.numel() > 0 for p in pipeline.transformer_2.parameters()), "Transformer_2 has no params!"
+        hidden_states: torch.Tensor = call.kwargs['hidden_states']
+        hidden_states_transposed = hidden_states.transpose(-1, -2).contiguous()
 
-        # --- AUTOQUANT + COMPILE ---
-        compiled_transformer = torchao.autoquant(
-            torch.compile(pipeline.transformer, mode='max-autotune'),
-            qtensor_class_list=DEFAULT_INT4_AUTOQUANT_CLASS_LIST
+        if hidden_states.shape[-1] > hidden_states.shape[-2]:
+            hidden_states_landscape = hidden_states
+            hidden_states_portrait = hidden_states_transposed
+        else:
+            hidden_states_landscape = hidden_states_transposed
+            hidden_states_portrait = hidden_states
+
+        print("[compile_transformer] Exporting transformer landscape model")
+        exported_landscape_1 = torch.export.export(
+            mod=pipeline.transformer,
+            args=call.args,
+            kwargs={**call.kwargs, 'hidden_states': hidden_states_landscape},
+            dynamic_shapes=dynamic_shapes,
         )
-        compiled_transformer_2 = torchao.autoquant(
-            torch.compile(pipeline.transformer_2, mode='max-autotune'),
-            qtensor_class_list=DEFAULT_INT4_AUTOQUANT_CLASS_LIST
+        torch.cuda.synchronize()
+
+        print("[compile_transformer] Exporting transformer portrait model")
+        exported_portrait_2 = torch.export.export(
+            mod=pipeline.transformer_2,
+            args=call.args,
+            kwargs={**call.kwargs, 'hidden_states': hidden_states_portrait},
+            dynamic_shapes=dynamic_shapes,
         )
+        torch.cuda.synchronize()
+
+        print("[compile_transformer] Compiling models with AoT compilation")
+        compiled_landscape_1 = aoti_compile(exported_landscape_1, INDUCTOR_CONFIGS)
+        compiled_portrait_2 = aoti_compile(exported_portrait_2, INDUCTOR_CONFIGS)
 
-        # --- PATCH FOR LANDSCAPE/PORTRAIT ---
-        def combined_transformer_1(*a, **k):
-            if k['hidden_states'].shape[-1] > k['hidden_states'].shape[-2]:
-                return compiled_transformer(*a, **k)
-            k_mod = dict(k)
-            k_mod['hidden_states'] = hidden_states_transposed
-            return compiled_transformer(*a, **k_mod)
+        compiled_landscape_2 = ZeroGPUCompiledModel(compiled_landscape_1.archive_file, compiled_portrait_2.weights)
+        compiled_portrait_1 = ZeroGPUCompiledModel(compiled_portrait_2.archive_file, compiled_landscape_1.weights)
 
-        def combined_transformer_2(*a, **k):
-            if k['hidden_states'].shape[-1] > k['hidden_states'].shape[-2]:
-                return compiled_transformer_2(*a, **k)
-            k_mod = dict(k)
-            k_mod['hidden_states'] = hidden_states_transposed
-            return compiled_transformer_2(*a, **k_mod)
+        print("[compile_transformer] Compilation done")
+        return compiled_landscape_1, compiled_landscape_2, compiled_portrait_1, compiled_portrait_2
 
-        pipeline.transformer.forward = combined_transformer_1
-        pipeline.transformer_2.forward = combined_transformer_2
+    cl1, cl2, cp1, cp2 = compile_transformer()
 
-        # --- NOW drain parameters to save VRAM ---
-        drain_module_parameters(pipeline.transformer)
-        drain_module_parameters(pipeline.transformer_2)
+    def combined_transformer_1(*args, **kwargs):
+        hidden_states: torch.Tensor = kwargs['hidden_states']
+        if hidden_states.shape[-1] > hidden_states.shape[-2]:
+            return cl1(*args, **kwargs)
+        else:
+            return cp1(*args, **kwargs)
 
-        print("[compile_transformer] Transformers autoquantized, compiled, and patched")
-        return compiled_transformer, compiled_transformer_2
+    def combined_transformer_2(*args, **kwargs):
+        hidden_states: torch.Tensor = kwargs['hidden_states']
+        if hidden_states.shape[-1] > hidden_states.shape[-2]:
+            return cl2(*args, **kwargs)
+        else:
+            return cp2(*args, **kwargs)
 
-    cl1, cl2 = compile_transformer()
-    print("[optimize_pipeline_] Optimization complete")
+    pipeline.transformer.forward = combined_transformer_1
+    drain_module_parameters(pipeline.transformer)
+    pipeline.transformer_2.forward = combined_transformer_2
+    drain_module_parameters(pipeline.transformer_2)
 
+    print("[optimize_pipeline_] Optimization complete")