x = x.to(torch.float16) # bfloat16

third_party/VideoLLaMA2/videollama2/model/projector.py

@@ -217,6 +217,7 @@ class STCConnector(nn.Module):
 
         x = einops.rearrange(x, "b d t h w -> (b t) d h w")
         # 1. the first stage of the adapter
+        x = x.to(torch.float16)  # bfloat16
         x = self.s1(x)
         x = einops.rearrange(x, "(b t) d h w -> b d t h w", t=t)
         # 2. downsampler