Fix issue where text encoder was not fully unloaded in some instances

toolkit/unloader.py

@@ -47,6 +47,7 @@ def unload_text_encoder(model: "BaseModel"):
             if hasattr(pipe, "text_encoder"):
                 te = FakeTextEncoder(device=model.device_torch, dtype=model.torch_dtype)
                 text_encoder_list.append(te)
+                pipe.text_encoder.to('cpu')
                 pipe.text_encoder = te
 
             i = 2