if(epoch+1)%args.val_per_n_epoch==0:ifrank==0:# save modelckpt_path='/fsx/awesome/comma2k19_checkpoints/%s_%d.pth'%(args.model,epoch)torch.save({'epoch':epoch,'model_state_dict':model.module.state_dict(),'optimizer_state_dict':optimizer.state_dict(),'lr_scheduler_state_dict':lr_scheduler.state_dict(),'loss':total_loss,},ckpt_path)time.sleep(10)# wait for all processes to save the filewandb.save(ckpt_path)print('[Epoch %d] checkpoint saved at %s'%(epoch,ckpt_path))
Warning
LINK
You are about to visit a link which has been flagged with the above content warnings. Do you wish to continue?