training/mono/model/losses/HDNL.py

import torch
import torch.nn as nn

class HDNLoss(nn.Module):
    """
    Hieratical depth normalization loss.
    loss = MAE((d-median(d)/s - (d'-median(d'))/s'), s = mean(d- median(d))
    """
    def __init__(self, loss_weight=1, grid=3, data_type=['sfm', 'stereo', 'lidar'], **kwargs):
        super(HDNLoss, self).__init__()
        self.loss_weight = loss_weight
        self.grid = grid
        self.data_type = data_type
    
    def get_hierachy_masks(self, grid, depth_gt, mask_valid):

        batch_map_grid = []
        for mask_index in range(depth_gt.shape[0]):
            depth_map = depth_gt[mask_index]
            valid_map = mask_valid[mask_index]

            # print (depth_map[valid_map].view(-1).shape)
            if depth_map[valid_map].numel() == 0:
                map_grid_list = [valid_map for _ in range(2 ** (grid) - 1)]
            else:
                valid_values = depth_map[valid_map]

                max_d = valid_values.max()
                min_d = valid_values.min()

                anchor_power = [(1 / 2) ** (i) for i in range(grid)]
                anchor_power.reverse()

                map_grid_list = []
                for anchor in anchor_power:
                    # range
                    for i in range(int(1 / anchor)):
                        mask_new = (depth_map >= min_d + (max_d - min_d) * i * anchor) & (
                                    depth_map < min_d + (max_d - min_d) * (i + 1) * anchor+1e-30)
                        # print (f'[{i*anchor},{(i+1)*anchor}]')
                        mask_new = mask_new & valid_map
                        map_grid_list.append(mask_new)
            map_grid_list = torch.stack(map_grid_list, dim=0)
            batch_map_grid.append(map_grid_list)
        batch_map_grid = torch.stack(batch_map_grid, dim=1)
        return batch_map_grid
    
    def ssi_mae(self, prediction, target, mask_valid):
        B, C, H, W = target.shape
        prediction_nan = prediction.clone()
        target_nan = target.clone()
        prediction_nan[~mask_valid] = float('nan')
        target_nan[~mask_valid] = float('nan')

        valid_pixs = mask_valid.reshape((B, C,-1)).sum(dim=2, keepdims=True) + 1e-10
        valid_pixs = valid_pixs[:, :, :, None]

        gt_median = target_nan.reshape((B, C,-1)).nanmedian(2, keepdims=True)[0].unsqueeze(-1) # [b,c,h,w]
        gt_median[torch.isnan(gt_median)] = 0
        gt_diff = (torch.abs(target - gt_median) * mask_valid).reshape((B, C, -1))
        gt_s = gt_diff.sum(dim=2)[:, :, None, None] / valid_pixs
        gt_trans = (target - gt_median) / (gt_s + 1e-8)

        pred_median = prediction_nan.reshape((B, C,-1)).nanmedian(2, keepdims=True)[0].unsqueeze(-1) # [b,c,h,w]
        pred_median[torch.isnan(pred_median)] = 0
        pred_diff = (torch.abs(prediction - pred_median) * mask_valid).reshape((B, C, -1))
        pred_s = pred_diff.sum(dim=2)[:, :, None, None] / valid_pixs
        pred_trans = (prediction - pred_median) / (pred_s + 1e-8)

        loss = torch.sum(torch.abs(gt_trans - pred_trans)*mask_valid) / (torch.sum(mask_valid) + 1e-8)
        return pred_trans, gt_trans, loss

    def forward(self, prediction, target, mask=None, **kwargs):
        """
        Calculate loss.
        """
        B, C, H, W = target.shape
        hierachy_masks = self.get_hierachy_masks(self.grid, target, mask)
        hierachy_masks_shape = hierachy_masks.reshape(-1, C, H, W)    
        prediction_hie = prediction.unsqueeze(0).repeat(hierachy_masks.shape[0], 1, 1, 1, 1).reshape(-1, C, H, W)     

        target_hie = target.unsqueeze(0).repeat(hierachy_masks.shape[0], 1, 1, 1, 1).reshape(-1, C, H, W)

        #_, _, loss = self.ssi_mae(prediction, target, mask)
        _, _, loss = self.ssi_mae(prediction_hie, target_hie, hierachy_masks_shape)
        return loss * self.loss_weight
    
if __name__ == '__main__':
    ssil = HDNLoss()
    pred = torch.rand((2, 1, 256, 256)).cuda()
    gt = torch.rand((2, 1, 256, 256)).cuda()#torch.zeros_like(pred).cuda() #
    gt[:, :, 100:256, 0:100] = -1
    mask = gt > 0
    out = ssil(pred, gt, mask)
    print(out)