pokouqiege/models/line_detect/trainer.py

import os
import time
from datetime import datetime

import cv2
import numpy as np
import torch
import torchvision
from PIL.ImageDraw import ImageDraw
from matplotlib import pyplot as plt
from mpmath import polar
from scipy.ndimage import gaussian_filter
from torch.optim.lr_scheduler import ReduceLROnPlateau
from torch.utils.tensorboard import SummaryWriter

from libs.vision_libs.utils import draw_bounding_boxes, draw_keypoints
from models.base.base_model import BaseModel
from models.base.base_trainer import BaseTrainer
from models.config.config_tool import read_yaml
from models.line_detect.line_dataset import LineDataset
import torch.nn.functional as F
from torchvision.transforms import functional as TF

from tools import utils

import matplotlib as mpl

from utils.data_process.show_prams import print_params

cmap = plt.get_cmap("jet")
norm = mpl.colors.Normalize(vmin=0.4, vmax=1.0)
sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm)
sm.set_array([])


def _loss(losses):
    total_loss = 0
    for i in losses.keys():
        if i != "loss_wirepoint":
            total_loss += losses[i]
        else:
            loss_labels = losses[i]["losses"]
    loss_labels_k = list(loss_labels[0].keys())
    for j, name in enumerate(loss_labels_k):
        loss = loss_labels[0][name].mean()
        total_loss += loss
    return total_loss


def c(x):
    return sm.to_rgba(x)


device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')


def draw_ellipses_on_image(image, masks_pred, threshold=0.5, color=(0, 255, 0), thickness=2):

    # Step 1: 标准化 masks_pred to [N, H, W]
    if masks_pred.ndim == 4:
        if masks_pred.shape[1] == 1:
            masks_pred = masks_pred.squeeze(1)  # [N, 1, H, W] -> [N, H, W]
        else:
            raise ValueError(f"Expected channel=1 in masks_pred, got shape {masks_pred.shape}")
    elif masks_pred.ndim != 3:
        raise ValueError(f"masks_pred must be 3D (N, H, W) or 4D (N, 1, H, W), got {masks_pred.shape}")

    N, H_mask, W_mask = masks_pred.shape
    C, H_img, W_img = image.shape

    # Step 2: Resize masks to original image size using bilinear interpolation
    masks_resized = F.interpolate(
        masks_pred.unsqueeze(1).float(),  # [N, 1, H_mask, W_mask]
        size=(H_img, W_img),
        mode='bilinear',
        align_corners=False
    ).squeeze(1)  # [N, H_img, W_img]

    # Step 3: Convert image to numpy RGB
    img_tensor = image.detach().cpu()
    if img_tensor.max() <= 1.0:
        img_np = (img_tensor * 255).byte().numpy()  # [3, H, W]
    else:
        img_np = img_tensor.byte().numpy()
    img_rgb = np.transpose(img_np, (1, 2, 0))  # [H, W, 3]
    img_out = img_rgb.copy()

    ellipses_info_list = []

    # Step 4: Process each mask
    for mask in masks_resized:
        mask_cpu = mask.detach().cpu()
        mask_prob = torch.sigmoid(mask_cpu) if mask_cpu.min() < 0 else mask_cpu
        binary = (mask_prob > threshold).numpy().astype(np.uint8) * 255  # [H_img, W_img]

        contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
        if contours:
            largest_contour = max(contours, key=cv2.contourArea)
            if len(largest_contour) >= 5:
                try:
                    ellipse = cv2.fitEllipse(largest_contour)
                    (center_x, center_y),( a,b),angle = ellipse

                    contour_points=largest_contour.reshape(-1,2)
                    points_centered=contour_points-np.array([center_x, center_y])

                    angle_red=np.deg2rad(angle)
                    rot_matrix=np.array([[np.cos(angle_red), -np.sin(angle_red)],[-np.sin(angle_red), np.cos(angle_red)]])
                    points_rotated=np.dot(points_centered, rot_matrix.T)

                    polar_angles=np.arctan2(points_rotated[:,1],points_rotated[:,0])

                    min_angle_idx=np.argmin(polar_angles)
                    max_angle_idx=np.argmax(polar_angles)

                    arc_start=tuple(contour_points[min_angle_idx].astype(int))
                    arc_end=tuple(contour_points[max_angle_idx].astype(int))

                    img_bgr = cv2.cvtColor(img_out, cv2.COLOR_RGB2BGR)
                    cv2.ellipse(img_bgr, ellipse, color=color, thickness=thickness)

                    cv2.circle(img_bgr, arc_start, 2, color=(0,0,255), thickness=-1)
                    cv2.circle(img_bgr, arc_end, 2, color=(255,0,0), thickness=-1)

                    img_out = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)

                    ellipses_info_list.append({"ellipse":ellipse,"arc_start":arc_start,"arc_end":arc_end})
                except cv2.error as e:
                    print(f"Warning: Failed to fit ellipse: {e}")

    return img_out, ellipses_info_list


def fit_circle(points):
    """
    Fit a circle to a set of points (at least 3).

    Args:
        points: torch.Tensor 或 numpy array, shape (N, 2)

    Returns:
        center (cx, cy), radius r
    """
    # 如果是 torch.Tensor，先转为 numpy
    if isinstance(points, torch.Tensor):
        if points.dim() == 3:
            points = points[0]  # 去掉 batch 维度
        points = points.detach().cpu().numpy()

    if not (isinstance(points, np.ndarray) and points.ndim == 2 and points.shape[1] == 2):
        raise ValueError(f"Expected points shape (N, 2), got {points.shape}")

    x = points[:, 0].astype(float)
    y = points[:, 1].astype(float)

    # 确保 A 是二维数组
    A = np.column_stack((x, y, np.ones_like(x)))  # 使用 column_stack 代替 stack 可能更清晰
    b = -(x ** 2 + y ** 2)

    try:
        sol, residuals, rank, s = np.linalg.lstsq(A, b, rcond=None)
    except np.linalg.LinAlgError as e:
        print(f"Linear algebra error occurred: {e}")
        raise ValueError("Could not fit circle to points.")

    D, E, F = sol

    cx = -D / 2.0
    cy = -E / 2.0
    r = np.sqrt(cx ** 2 + cy ** 2 - F)

    return (cx, cy), r


from PIL import ImageDraw, Image
import io


def draw_el(all, background_img):
    """
    all = [x_center, y_center, a, b, theta, x1, y1, x2, y2]
        theta: ellipse rotation (degrees)
        (x1, y1): start point
        (x2, y2): end point
    """

    if isinstance(all, torch.Tensor):
        all = all.cpu().numpy()

    # Unpack parameters
    cx, cy, a, b, theta_deg, x1, y1, x2, y2 = all

    # cx = cx / 672 * 2000
    # cy = cy / 672 * 2000
    # # a = a / 672 * 2000
    # # b = b / 672 * 2000
    # x1 = x1 / 672 * 2000
    # y1 = y1 / 672 * 2000
    # x2 = x2 / 672 * 2000
    # y2 = y2 / 672 * 2000



    theta = np.radians(theta_deg)

    # ====== Draw ellipse ======
    phi = np.linspace(0, np.pi * 2, 500)
    x_ellipse = cx + a * np.cos(phi) * np.cos(theta) - b * np.sin(phi) * np.sin(theta)
    y_ellipse = cy + a * np.cos(phi) * np.sin(theta) + b * np.sin(phi) * np.cos(theta)

    # ====== Draw image ======
    plt.figure(figsize=(10, 10))
    plt.imshow(background_img)

    # Ellipse
    plt.plot(x_ellipse, y_ellipse, 'b-', linewidth=2)

    # Center
    plt.plot(cx, cy, 'ko', markersize=8)

    # Start & End points (now real coordinates)
    plt.plot(x1, y1, 'ro', markersize=10)
    plt.plot(x2, y2, 'go', markersize=10)

    # ====== Convert to tensor ======
    buf = io.BytesIO()
    plt.savefig(buf, format='png', bbox_inches='tight')
    buf.seek(0)
    result_img = Image.open(buf).convert('RGB')
    img_tensor = torch.from_numpy(np.array(result_img)).permute(2, 0, 1)
    plt.close()

    return img_tensor


# from PIL import ImageDraw, Image
# import io
# # 绘制椭圆
# def draw_el(all, background_img):
#     # 解析椭圆参数
#     if isinstance(all, torch.Tensor):
#         all = all.cpu().numpy()
#     print_params(all)
#     x, y, a, b, q, q1, q2 = all
#     theta = np.radians(q)
#     phi1 = np.radians(q1)  # 第一个点的参数角
#     phi2 = np.radians(q2)  # 第二个点的参数角
#
#     # 生成椭圆上的点
#     phi = np.linspace(0, 2 * np.pi, 500)
#     x_ellipse = x + a * np.cos(phi) * np.cos(theta) - b * np.sin(phi) * np.sin(theta)
#     y_ellipse = y + a * np.cos(phi) * np.sin(theta) + b * np.sin(phi) * np.cos(theta)
#
#     # 计算两个指定点的坐标
#     def param_to_point(phi, xc, yc, a, b, theta):
#         x = xc + a * np.cos(phi) * np.cos(theta) - b * np.sin(phi) * np.sin(theta)
#         y = yc + a * np.cos(phi) * np.sin(theta) + b * np.sin(phi) * np.cos(theta)
#         return x, y
#
#     P1 = param_to_point(phi1, x, y, a, b, theta)
#     P2 = param_to_point(phi2, x, y, a, b, theta)
#
#     # 创建画布并显示背景图片（使用传入的background_img，shape为[H, W, C]）
#     plt.figure(figsize=(10, 10))
#     plt.imshow(background_img)  # 直接显示背景图
#
#     # 绘制椭圆及相关元素
#     plt.plot(x_ellipse, y_ellipse, 'b-', linewidth=2)
#     plt.plot(x, y, 'ko', markersize=8)
#     plt.plot(P1[0], P1[1], 'ro', markersize=10)
#     plt.plot(P2[0], P2[1], 'go', markersize=10)

# 转换为TensorBoard所需的张量格式 [C, H, W]
# buf = io.BytesIO()
# plt.savefig(buf, format='png', bbox_inches='tight')
# buf.seek(0)
# result_img = Image.open(buf).convert('RGB')
# img_tensor = torch.from_numpy(np.array(result_img)).permute(2, 0, 1)
# plt.close()
#
# return img_tensor
# 由低到高蓝黄红
def draw_lines_with_scores(tensor_image, lines, scores, width=3, cmap='viridis'):
    """
    根据得分对线段着色并绘制
    :param tensor_image: (3, H, W) uint8 图像
    :param lines: (N, 2, 2) 每条线 [ [x1,y1], [x2,y2] ]
    :param scores: (N,) 每条线的得分，范围 [0, 1]
    :param width: 线宽
    :param cmap: matplotlib colormap 名称，例如 'viridis', 'jet', 'coolwarm'
    :return: (3, H, W) uint8 画好线的图像
    """
    assert tensor_image.dtype == torch.uint8
    assert tensor_image.shape[0] == 3
    assert lines.shape[0] == scores.shape[0]

    # 准备色图
    colormap = plt.get_cmap(cmap)
    colors = (colormap(scores.cpu().numpy())[:, :3] * 255).astype('uint8')  # 去掉 alpha 通道

    # 转为 PIL 画图
    image_pil = TF.to_pil_image(tensor_image)
    draw = ImageDraw.Draw(image_pil)

    for line, color in zip(lines, colors):
        start = tuple(map(float, line[0][:2].tolist()))
        end = tuple(map(float, line[1][:2].tolist()))
        draw.line([start, end], fill=tuple(color), width=width)

    return (torchvision.transforms.functional.to_tensor(image_pil) * 255).to(torch.uint8)


class Trainer(BaseTrainer):
    def __init__(self, model=None, **kwargs):
        super().__init__(model, device, **kwargs)
        self.model = model
        # print(f'kwargs:{kwargs}')
        self.init_params(**kwargs)

    def init_params(self, **kwargs):
        if kwargs != {}:
            print(f'train_params:{kwargs["train_params"]}')
            self.freeze_config = kwargs['train_params']['freeze_params']
            print(f'freeze_config:{self.freeze_config}')
            self.dataset_path = kwargs['io']['datadir']
            self.data_type = kwargs['io']['data_type']
            self.batch_size = kwargs['train_params']['batch_size']
            self.num_workers = kwargs['train_params']['num_workers']
            self.logdir = kwargs['io']['logdir']
            self.resume_from = kwargs['train_params']['resume_from']
            self.optim = ''
            self.train_result_ptath = os.path.join(self.logdir, datetime.now().strftime("%Y%m%d_%H%M%S"))
            self.wts_path = os.path.join(self.train_result_ptath, 'weights')
            self.tb_path = os.path.join(self.train_result_ptath, 'logs')
            self.writer = SummaryWriter(self.tb_path)
            self.last_model_path = os.path.join(self.wts_path, 'last.pth')
            self.best_train_model_path = os.path.join(self.wts_path, 'best_train.pth')
            self.best_val_model_path = os.path.join(self.wts_path, 'best_val.pth')
            self.max_epoch = kwargs['train_params']['max_epoch']
            self.augmentation = kwargs['train_params']["augmentation"]

    def move_to_device(self, data, device):
        if isinstance(data, (list, tuple)):
            return type(data)(self.move_to_device(item, device) for item in data)
        elif isinstance(data, dict):
            return {key: self.move_to_device(value, device) for key, value in data.items()}
        elif isinstance(data, torch.Tensor):
            return data.to(device)
        else:
            return data  # 对于非张量类型的数据不做任何改变

    def freeze_params(self, model):
        """根据配置冻结模型参数"""
        default_config = {
            'backbone': True,  # 冻结 backbone
            'rpn': False,  # 不冻结 rpn
            'roi_heads': {
                'box_head': False,
                'box_predictor': False,
                'line_head': False,
                'line_predictor': {
                    'fc1': False,
                    'fc2': {
                        '0': False,
                        '2': False,
                        '4': False
                    }
                }
            }
        }

        # 更新默认配置
        default_config.update(self.freeze_config)
        config = default_config

        print("\n===== Parameter Freezing Configuration =====")
        for name, module in model.named_children():
            if name in config:
                if isinstance(config[name], bool):
                    for param in module.parameters():
                        param.requires_grad = not config[name]
                    print(f"{'Frozen' if config[name] else 'Trainable'} module: {name}")

                elif isinstance(config[name], dict):
                    for subname, submodule in module.named_children():
                        if subname in config[name]:
                            if isinstance(config[name][subname], bool):
                                for param in submodule.parameters():
                                    param.requires_grad = not config[name][subname]
                                print(
                                    f"{'Frozen' if config[name][subname] else 'Trainable'} submodule: {name}.{subname}")

                            elif isinstance(config[name][subname], dict):
                                for subsubname, subsubmodule in submodule.named_children():
                                    if subsubname in config[name][subname]:
                                        for param in subsubmodule.parameters():
                                            param.requires_grad = not config[name][subname][subsubname]
                                        print(
                                            f"{'Frozen' if config[name][subname][subsubname] else 'Trainable'} sub-submodule: {name}.{subname}.{subsubname}")

        # 打印参数统计
        total_params = sum(p.numel() for p in model.parameters())
        trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
        print(f"\nTotal Parameters: {total_params:,}")
        print(f"Trainable Parameters: {trainable_params:,}")
        print(f"Frozen Parameters: {total_params - trainable_params:,}")

    def load_best_model(self, model, optimizer, save_path, device):
        if os.path.exists(save_path):
            checkpoint = torch.load(save_path, map_location=device)
            model.load_state_dict(checkpoint['model_state_dict'])
            if optimizer is not None:
                optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
            epoch = checkpoint['epoch']
            loss = checkpoint['loss']
            print(f"Loaded best model from {save_path} at epoch {epoch} with loss {loss:.4f}")
        else:
            print(f"No saved model found at {save_path}")
        return model, optimizer

    def writer_predict_result(self, img, result, epoch, ):
        img = img.cpu().detach()
        im = img.permute(1, 2, 0)  # [512, 512, 3]
        self.writer.add_image("z-ori", im, epoch, dataformats="HWC")

        boxed_image = draw_bounding_boxes((img * 255).to(torch.uint8), result["boxes"],
                                          colors="yellow", width=1)

        # plt.imshow(boxed_image.permute(1, 2, 0).detach().cpu().numpy())
        # plt.show()

        self.writer.add_image("z-obj", boxed_image.permute(1, 2, 0), epoch, dataformats="HWC")

        if 'points' in result:
            keypoint_img = draw_keypoints(boxed_image, result['points'], colors='red', width=3)

            self.writer.add_image("z-output", keypoint_img, epoch)
        # print("lines shape:", result['lines'].shape)

        if 'lines' in result:
            # 用自己写的函数画线段
            # line_image = draw_lines(boxed_image, result['lines'], color='red', width=3)
            print(f"shape of linescore:{result['lines_scores'].shape}")
            scores = result['lines_scores'].mean(dim=1)  # shape: [31]

            line_image = draw_lines_with_scores((img * 255).to(torch.uint8), result['lines'], scores, width=3,
                                                cmap='jet')

            self.writer.add_image("z-output_line", line_image.permute(1, 2, 0), epoch, dataformats="HWC")

        if 'arcs' in result:
            arcs = result['arcs'][0]
            print(f'arcs in draw:{arcs}')

            ellipse_img = draw_el(arcs, background_img=im)

            # img_rgb = cv2.cvtColor(img_np, cv2.COLOR_BGR2RGB)
            #
            # img_tensor =torch.tensor(img_rgb)
            # img_tensor = np.transpose(img_tensor)

            self.writer.add_image('z-out-arc', ellipse_img, global_step=epoch)

        if 'ins_masks' in result:
            # points=result['circles']
            # points=points.squeeze(1)
            ppp = result['ins_masks']
            bbb = result['boxes']
            print(f'boxes shape:{bbb.shape}')
            print(f'ppp:{ppp.shape}')
            ins_masks = result['ins_masks']
            ins_masks = ins_masks.squeeze(1)
            print(f'ins_masks shape:{ins_masks.shape}')
            features = result['features']

            circle_image = img.cpu().numpy().transpose((1, 2, 0))  # CHW -> HWC
            circle_image = (circle_image * 255).clip(0, 255).astype(np.uint8)

            sum_mask = ins_masks.sum(dim=0, keepdim=True)
            sum_mask = sum_mask / (sum_mask.max() + 1e-8)

            # keypoint_img = draw_keypoints((img * 255).to(torch.uint8), points, colors='red', width=3)
            self.writer.add_image('z-ins-masks', sum_mask.squeeze(0), global_step=epoch)

            result_imgs, _ = draw_ellipses_on_image(img, ins_masks, threshold=0.5)
            self.writer.add_image('z-out-ellipses', result_imgs, dataformats='HWC', global_step=epoch)

            features = self.apply_gaussian_blur_to_tensor(features, sigma=3)
            self.writer.add_image('z-feature', features, global_step=epoch)

            # cv2.imshow('arc', img_rgb)
            # cv2.waitKey(1000000)

    def normalize_tensor(self, tensor):
        """Normalize tensor to [0, 1]"""
        min_val = tensor.min()
        max_val = tensor.max()
        return (tensor - min_val) / (max_val - min_val)

    def apply_gaussian_blur_to_tensor(self, feature_map, sigma=3):
        """
        Apply Gaussian blur to a feature map and convert it into an RGB heatmap.

        :param feature_map: Tensor of shape (H, W) or (1, H, W)
        :param sigma: Standard deviation for Gaussian kernel
        :return: Tensor of shape (3, H, W) representing the RGB heatmap
        """
        if feature_map.dim() == 3:
            if feature_map.shape[0] != 1:
                raise ValueError("Only single-channel feature map supported.")
            feature_map = feature_map.squeeze(0)

        # Normalize to [0, 1]
        normalized_feat = self.normalize_tensor(feature_map).cpu().numpy()

        # Apply Gaussian blur
        blurred_feat = gaussian_filter(normalized_feat, sigma=sigma)

        # Convert to colormap (e.g., 'jet')
        colormap = plt.get_cmap('jet')
        colored = colormap(blurred_feat)  # shape: (H, W, 4) RGBA

        # Convert to (3, H, W), drop alpha channel
        colored_rgb = colored[:, :, :3]  # (H, W, 3)
        colored_tensor = torch.from_numpy(colored_rgb).permute(2, 0, 1)  # (3, H, W)

        return colored_tensor.float()

    def writer_loss(self, losses, epoch, phase='train'):
        try:
            for key, value in losses.items():
                if key == 'loss_wirepoint':
                    for subdict in losses['loss_wirepoint']['losses']:
                        for subkey, subvalue in subdict.items():
                            self.writer.add_scalar(f'{phase}/loss/{subkey}',
                                                   subvalue.item() if hasattr(subvalue, 'item') else subvalue,
                                                   epoch)
                elif isinstance(value, torch.Tensor):
                    self.writer.add_scalar(f'{phase}/loss/{key}', value.item(), epoch)
        except Exception as e:
            print(f"TensorBoard logging error: {e}")

    def train_from_cfg(self, model: BaseModel, cfg, freeze_config=None):  # 新增：支持传入冻结配置
        cfg = read_yaml(cfg)
        # print(f'cfg:{cfg}')
        # self.freeze_config = freeze_config or {}  # 更新冻结配置

        self.train(model, **cfg)

    def train(self, model, **kwargs):

        self.init_params(**kwargs)

        dataset_train = LineDataset(dataset_path=self.dataset_path, augmentation=self.augmentation,
                                    data_type=self.data_type, dataset_type='train')
        dataset_val = LineDataset(dataset_path=self.dataset_path, augmentation=self.augmentation,
                                  data_type=self.data_type, dataset_type='val')

        train_sampler = torch.utils.data.RandomSampler(dataset_train)
        val_sampler = torch.utils.data.RandomSampler(dataset_val)
        train_batch_sampler = torch.utils.data.BatchSampler(train_sampler, batch_size=self.batch_size, drop_last=True)
        val_batch_sampler = torch.utils.data.BatchSampler(val_sampler, batch_size=self.batch_size, drop_last=True)
        train_collate_fn = utils.collate_fn
        val_collate_fn = utils.collate_fn

        data_loader_train = torch.utils.data.DataLoader(
            dataset_train, batch_sampler=train_batch_sampler, num_workers=self.num_workers, collate_fn=train_collate_fn
        )
        data_loader_val = torch.utils.data.DataLoader(
            dataset_val, batch_sampler=val_batch_sampler, num_workers=self.num_workers, collate_fn=val_collate_fn
        )

        model.to(device)

        optimizer = torch.optim.Adam(
            filter(lambda p: p.requires_grad, model.parameters()),
            lr=kwargs['train_params']['optim']['lr'],
            weight_decay=kwargs['train_params']['optim']['weight_decay'],

        )

        # model, optimizer = self.load_best_model(model, optimizer,
        #                                         r"/home/limin/PycharmProjects/pokouqiege/1126/MultiVisionModels/models/line_detect/train_results/20251213_145349/weights/best_val.pth",
        #                                         device)
        # scheduler = StepLR(optimizer, step_size=10, gamma=0.1)
        scheduler = ReduceLROnPlateau(optimizer, 'min', patience=30)

        for epoch in range(self.max_epoch):
            print(f"train epoch:{epoch}")

            model, epoch_train_loss = self.one_epoch(model, data_loader_train, epoch, optimizer)
            scheduler.step(epoch_train_loss)

            # ========== Validation ==========
            with torch.no_grad():
                model, epoch_val_loss = self.one_epoch(model, data_loader_val, epoch, optimizer, phase='val')
                scheduler.step(epoch_val_loss)

            if epoch == 0:
                best_train_loss = epoch_train_loss
                best_val_loss = epoch_val_loss

            self.save_last_model(model, self.last_model_path, epoch, optimizer)
            best_train_loss = self.save_best_model(model, self.best_train_model_path, epoch, epoch_train_loss,
                                                   best_train_loss,
                                                   optimizer)
            best_val_loss = self.save_best_model(model, self.best_val_model_path, epoch, epoch_val_loss, best_val_loss,
                                                 optimizer)

    def one_epoch(self, model, data_loader, epoch, optimizer, phase='train'):
        if phase == 'train':
            model.train()
        if phase == 'val':
            model.eval()

        total_loss = 0
        epoch_step = 0
        global_step = epoch * len(data_loader)
        for imgs, targets in data_loader:
            imgs = self.move_to_device(imgs, device)
            targets = self.move_to_device(targets, device)
            if phase == 'val':
                result, loss_dict = model(imgs, targets)
                losses = sum(loss_dict.values())

                print(f'val losses:{losses}')
                # print(f'val result:{result}')
            else:
                loss_dict = model(imgs, targets)
                losses = sum(loss_dict.values())
                print(f'train losses:{losses}')

            # loss = _loss(losses)
            loss = losses
            total_loss += loss.item()
            if phase == 'train':
                optimizer.zero_grad()
                loss.backward()
                torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0)
                optimizer.step()
            self.writer_loss(loss_dict, global_step, phase=phase)
            global_step += 1

            if epoch_step == 0 and phase == 'val':
                t_start = time.time()
                print(f'start to predict:{t_start}')
                result = model(self.move_to_device(imgs, self.device))
                # print(f'result:{result}')
                t_end = time.time()
                print(f'predict used:{t_end - t_start}')
                from utils.data_process.show_prams import print_params
                print_params(imgs[0], result[0], epoch)
                self.writer_predict_result(img=imgs[0], result=result[0], epoch=epoch)
                epoch_step += 1

        avg_loss = total_loss / len(data_loader)
        print(f'{phase}/loss epoch{epoch}:{avg_loss:4f}')
        self.writer.add_scalar(f'loss/{phase}', avg_loss, epoch)
        return model, avg_loss

    def save_best_model(self, model, save_path, epoch, current_loss, best_loss, optimizer=None):
        os.makedirs(os.path.dirname(save_path), exist_ok=True)

        if current_loss <= best_loss:
            checkpoint = {
                'epoch': epoch,
                'model_state_dict': model.state_dict(),
                'loss': current_loss
            }
            if optimizer is not None:
                checkpoint['optimizer_state_dict'] = optimizer.state_dict()

            torch.save(checkpoint, save_path)
            print(f"Saved best model at epoch {epoch} with loss {current_loss:.4f}")

            return current_loss

        return best_loss

    def save_last_model(self, model, save_path, epoch, optimizer=None):

        if os.path.exists(f'{self.wts_path}/last.pt'):
            os.remove(f'{self.wts_path}/last.pt')

        os.makedirs(os.path.dirname(save_path), exist_ok=True)

        checkpoint = {
            'epoch': epoch,
            'model_state_dict': model.state_dict(),
        }

        if optimizer is not None:
            checkpoint['optimizer_state_dict'] = optimizer.state_dict()

        torch.save(checkpoint, save_path)


if __name__ == '__main__':
    print('')