|
@@ -182,7 +182,8 @@ class Trainer(BaseTrainer):
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
last_model_path = os.path.join(wts_path, 'last.pth')
|
|
last_model_path = os.path.join(wts_path, 'last.pth')
|
|
|
- best_model_path = os.path.join(wts_path, 'best.pth')
|
|
|
|
|
|
|
+ best_train_model_path = os.path.join(wts_path, 'best_train.pth')
|
|
|
|
|
+ best_val_model_path = os.path.join(wts_path, 'best_val.pth')
|
|
|
global_step = 0
|
|
global_step = 0
|
|
|
|
|
|
|
|
for epoch in range(kwargs['optim']['max_epoch']):
|
|
for epoch in range(kwargs['optim']['max_epoch']):
|
|
@@ -201,22 +202,28 @@ class Trainer(BaseTrainer):
|
|
|
self.writer_loss(writer, losses, global_step)
|
|
self.writer_loss(writer, losses, global_step)
|
|
|
global_step += 1
|
|
global_step += 1
|
|
|
|
|
|
|
|
- avg_train_loss = total_train_loss / len(data_loader_train)
|
|
|
|
|
- if epoch == 0:
|
|
|
|
|
- best_loss = avg_train_loss
|
|
|
|
|
- writer.add_scalar('loss/train', avg_train_loss, epoch)
|
|
|
|
|
|
|
|
|
|
- if os.path.exists(f'{wts_path}/last.pt'):
|
|
|
|
|
- os.remove(f'{wts_path}/last.pt')
|
|
|
|
|
- save_last_model(model, last_model_path, epoch, optimizer)
|
|
|
|
|
- best_loss = save_best_model(model, best_model_path, epoch, avg_train_loss, best_loss, optimizer)
|
|
|
|
|
|
|
|
|
|
model.eval()
|
|
model.eval()
|
|
|
|
|
+ print(f'model.eval!!')
|
|
|
|
|
+ # ========== Validation ==========
|
|
|
|
|
+ total_val_loss = 0.0
|
|
|
with torch.no_grad():
|
|
with torch.no_grad():
|
|
|
for batch_idx, (imgs, targets) in enumerate(data_loader_val):
|
|
for batch_idx, (imgs, targets) in enumerate(data_loader_val):
|
|
|
t_start = time.time()
|
|
t_start = time.time()
|
|
|
print(f'start to predict:{t_start}')
|
|
print(f'start to predict:{t_start}')
|
|
|
- pred = model(self.move_to_device(imgs, self.device))
|
|
|
|
|
|
|
+
|
|
|
|
|
+ imgs = move_to_device(imgs, device)
|
|
|
|
|
+ targets = move_to_device(targets, device)
|
|
|
|
|
+ print(f'targets:{targets}')
|
|
|
|
|
+
|
|
|
|
|
+ losses = model(imgs, targets)
|
|
|
|
|
+ print(f'val losses:{losses}')
|
|
|
|
|
+ loss = _loss(losses)
|
|
|
|
|
+ total_val_loss += loss.item()
|
|
|
|
|
+
|
|
|
|
|
+ pred= model(self.move_to_device(imgs, self.device))
|
|
|
|
|
+
|
|
|
# print(f'pred:{pred}')
|
|
# print(f'pred:{pred}')
|
|
|
t_end = time.time()
|
|
t_end = time.time()
|
|
|
print(f'predict used:{t_end - t_start}')
|
|
print(f'predict used:{t_end - t_start}')
|
|
@@ -224,6 +231,27 @@ class Trainer(BaseTrainer):
|
|
|
show_line(imgs[0], pred, epoch, writer)
|
|
show_line(imgs[0], pred, epoch, writer)
|
|
|
break
|
|
break
|
|
|
|
|
|
|
|
|
|
+ avg_val_loss = total_val_loss / len(data_loader_val)
|
|
|
|
|
+ # print(f'avg_val_loss:{avg_val_loss}')
|
|
|
|
|
+
|
|
|
|
|
+ avg_train_loss = total_train_loss / len(data_loader_train)
|
|
|
|
|
+ print(f'avg_train_loss:{avg_train_loss}')
|
|
|
|
|
+ if epoch == 0:
|
|
|
|
|
+ best_train_loss = avg_train_loss
|
|
|
|
|
+ best_val_loss = avg_val_loss
|
|
|
|
|
+ writer.add_scalar('loss/train', avg_train_loss, epoch)
|
|
|
|
|
+
|
|
|
|
|
+ if os.path.exists(f'{wts_path}/last.pt'):
|
|
|
|
|
+ os.remove(f'{wts_path}/last.pt')
|
|
|
|
|
+ save_last_model(model, last_model_path, epoch, optimizer)
|
|
|
|
|
+ best_train_loss = save_best_model(model, best_train_model_path, epoch, avg_train_loss, best_train_loss,
|
|
|
|
|
+ optimizer)
|
|
|
|
|
+
|
|
|
|
|
+ best_val_loss = save_best_model(model, best_val_model_path, epoch, avg_val_loss, best_val_loss,
|
|
|
|
|
+ optimizer)
|
|
|
|
|
+ writer.add_scalar('loss/val', avg_val_loss, epoch)
|
|
|
|
|
+ print(f"Epoch {epoch} - Val Loss: {avg_val_loss:.4f}")
|
|
|
|
|
+
|
|
|
|
|
|
|
|
import torch
|
|
import torch
|
|
|
|
|
|