Fix layer offloaded adaln projection layer move

This commit is contained in:
Jaret Burkett 2026-08-06 08:10:20 -06:00
parent 139a38f5bd
commit 7309db4d74
1 changed files with 7 additions and 5 deletions

View File

@ -217,11 +217,13 @@ class MiniMaxH3AdalnProj(nn.Module):
def forward(self, temb: torch.Tensor):
if self.apply_silu:
temb = F.silu(temb)
x = F.linear(
temb.float(),
self.linear.weight.float(),
self.linear.bias.float() if self.linear.bias is not None else None,
)
# follow temb's device: the raw params may be CPU-resident under layer
# offloading / low_vram, and F.linear bypasses the paging hooks
weight = self.linear.weight.to(device=temb.device, dtype=torch.float32)
bias = self.linear.bias
if bias is not None:
bias = bias.to(device=temb.device, dtype=torch.float32)
x = F.linear(temb.float(), weight, bias)
x = x.view(x.shape[0] * self.modalities, self.expand * self.hidden)
return x.chunk(self.expand, dim=-1)