Update optimization.py

optimization.py

@@ -39,11 +39,12 @@ INDUCTOR_CONFIGS = {
 def optimize_pipeline_(pipeline: Callable[P, Any], *args: P.args, **kwargs: P.kwargs):
     print("[optimize_pipeline_] Starting pipeline optimization")
 
-    # Quantize and compile text encoder first (weight-only int8 quantization can be replaced by autoquant if preferred)
+    # Text encoder: move to CPU first, then quantize+compile to avoid early CUDA init
+    pipeline.text_encoder = pipeline.text_encoder.cpu()
     pipeline.text_encoder = torchao.autoquant(
         torch.compile(pipeline.text_encoder, mode='max-autotune'),
-        qtensor_class_list=DEFAULT_INT4_AUTOQUANT_CLASS_LIST  # or remove for default quant
-    )
+        qtensor_class_list=DEFAULT_INT4_AUTOQUANT_CLASS_LIST
+    ).to("cuda")
     print("[optimize_pipeline_] Text encoder autoquantized and compiled")
 
     @spaces.GPU(duration=1500)
@@ -67,7 +68,7 @@ def optimize_pipeline_(pipeline: Callable[P, Any], *args: P.args, **kwargs: P.kw
         pipeline.fuse_lora(adapter_names=["lightning_2"], lora_scale=1.0, components=["transformer_2"])
         pipeline.unload_lora_weights()
 
-        print("[compile_transformer] Running dummy forward pass to capture component call")
+        print("[compile_transformer] Running dummy forward pass to capture shapes")
         with torch.inference_mode():
             with capture_component_call(pipeline, 'transformer') as call:
                 pipeline(*args, **kwargs)
@@ -75,44 +76,36 @@ def optimize_pipeline_(pipeline: Callable[P, Any], *args: P.args, **kwargs: P.kw
         dynamic_shapes = tree_map_only((torch.Tensor, bool), lambda t: None, call.kwargs)
         dynamic_shapes |= TRANSFORMER_DYNAMIC_SHAPES
 
-        # Use autoquant + torch.compile on transformers
-        print("[compile_transformer] Autoquantizing and compiling transformer")
+        # Now that we're inside GPU context, move and compile transformers
+        pipeline.transformer = pipeline.transformer.to("cuda")
+        pipeline.transformer_2 = pipeline.transformer_2.to("cuda")
+
         compiled_transformer = torchao.autoquant(
             torch.compile(pipeline.transformer, mode='max-autotune'),
-            qtensor_class_list=DEFAULT_INT4_AUTOQUANT_CLASS_LIST,
+            qtensor_class_list=DEFAULT_INT4_AUTOQUANT_CLASS_LIST
         )
         compiled_transformer_2 = torchao.autoquant(
             torch.compile(pipeline.transformer_2, mode='max-autotune'),
-            qtensor_class_list=DEFAULT_INT4_AUTOQUANT_CLASS_LIST,
+            qtensor_class_list=DEFAULT_INT4_AUTOQUANT_CLASS_LIST
         )
 
+        # Patch forward with landscape/portrait logic
         hidden_states: torch.Tensor = call.kwargs['hidden_states']
         hidden_states_transposed = hidden_states.transpose(-1, -2).contiguous()
 
-        if hidden_states.shape[-1] > hidden_states.shape[-2]:
-            hidden_states_landscape = hidden_states
-            hidden_states_portrait = hidden_states_transposed
-        else:
-            hidden_states_landscape = hidden_states_transposed
-            hidden_states_portrait = hidden_states
-
-        # Replace forward with quantized & compiled versions, wrapped for shape dispatch
         def combined_transformer_1(*a, **k):
             if k['hidden_states'].shape[-1] > k['hidden_states'].shape[-2]:
                 return compiled_transformer(*a, **k)
-            else:
-                # Swap hidden states for portrait? Use transpose if needed.
-                k_mod = k.copy()
-                k_mod['hidden_states'] = hidden_states_portrait
-                return compiled_transformer(*a, **k_mod)
+            k_mod = k.copy()
+            k_mod['hidden_states'] = hidden_states_transposed
+            return compiled_transformer(*a, **k_mod)
 
         def combined_transformer_2(*a, **k):
             if k['hidden_states'].shape[-1] > k['hidden_states'].shape[-2]:
                 return compiled_transformer_2(*a, **k)
-            else:
-                k_mod = k.copy()
-                k_mod['hidden_states'] = hidden_states_portrait
-                return compiled_transformer_2(*a, **k_mod)
+            k_mod = k.copy()
+            k_mod['hidden_states'] = hidden_states_transposed
+            return compiled_transformer_2(*a, **k_mod)
 
         pipeline.transformer.forward = combined_transformer_1
         drain_module_parameters(pipeline.transformer)
@@ -121,8 +114,6 @@ def optimize_pipeline_(pipeline: Callable[P, Any], *args: P.args, **kwargs: P.kw
         drain_module_parameters(pipeline.transformer_2)
 
         print("[compile_transformer] Transformers autoquantized, compiled, and patched")
-
-        # Return compiled models for reference if needed
         return compiled_transformer, compiled_transformer_2
 
     cl1, cl2 = compile_transformer()