RomanNum3ral
/
ai-toolkit
mirror of https://github.com/ostris/ai-toolkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Optimizer Saving Update(BaseSDTrainProcess): 

Now maintains a backup of the previous optimizer state. When saving, the current optimizer.pt is moved to optimizer_prev.pt rather than being overwritten.
This commit is contained in:
avan 2026-02-22 11:27:04 +08:00
parent 19b7170e58
commit 5cd62b0a7d
1 changed files with 12 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
jobs/process/BaseSDTrainProcess.py
View File

@ -684,12 +684,23 @@ class BaseSDTrainProcess(BaseTrainProcess):
try:
filename = f'optimizer.pt'
file_path = os.path.join(self.save_root, filename)
backup_optimizer_path = os.path.join(self.save_root, 'optimizer_prev.pt')
# If optimizer.pt already exists, rename it to optimizer_prev.pt
if os.path.exists(file_path):
# If an old backup also exists, delete it first (to ensure a successful rename)
if os.path.exists(backup_optimizer_path):
os.remove(backup_optimizer_path)
os.rename(file_path, backup_optimizer_path)
print_acc(f"Existing optimizer.pt moved to optimizer_prev.pt")
try:
state_dict = unwrap_model(self.optimizer).state_dict()
except Exception as e:
state_dict = self.optimizer.state_dict()
torch.save(state_dict, file_path)
print_acc(f"Saved optimizer to {file_path}")
print_acc(f"Saved latest optimizer to {file_path}")
except Exception as e:
print_acc(e)
print_acc("Could not save optimizer")