<filename>models/networks/loss.py
import torch
from torch import nn


class GANLoss(nn.Module):
    """Define different GAN Discriminator's objectives.

    The GANLoss class abstracts away the need to create the target label tensor
    that has the same size as the input.
    """

    def __init__(self, loss_mode, which_net, which_D, target_real_label=1.0, target_fake_label=0.0):
        """ Initialize the GAN's Discriminator Loss class.

        Parameters:
            loss_mode (str) - - the type of GAN objective. It currently supports vanilla, lsgan, and wgangp.
            target_real_label (bool) - - label for a real image
            target_fake_label (bool) - - label of a fake image

        Note: Do not use sigmoid as the last layer of Discriminator.
        LSGAN needs no sigmoid. vanilla GANs will handle it with BCEWithLogitsLoss.
        """
        super(GANLoss, self).__init__()
        self.register_buffer('real_label', torch.tensor(target_real_label))
        self.register_buffer('fake_label', torch.tensor(target_fake_label))
        self.loss_mode = loss_mode
        self.which_net = which_net
        self.which_D = which_D

        if loss_mode == 'lsgan':
            self.loss = nn.MSELoss()
        elif loss_mode in ['vanilla', 'nsgan', 'rsgan']:
            self.loss = nn.BCEWithLogitsLoss()
        elif loss_mode in ['wgan', 'hinge']:
            self.loss = None
        else:
            raise NotImplementedError('gan mode %s not implemented' % loss_mode)

    def get_target_tensor(self, prediction, target_is_real):
        """Create label tensors with the same size as the input.
        Parameters:
            prediction (tensor) - - tpyically the prediction from a discriminator
            target_is_real (bool) - - if the ground truth label is for real images or fake images
        Returns:
            A label tensor filled with ground truth label, and with the size of the input
        """
        if target_is_real:
            target_tensor = self.real_label
        else:
            target_tensor = self.fake_label
        return target_tensor.expand_as(prediction)

    def G_loss(self, Dfake, Dreal):
        real_tensor = self.get_target_tensor(Dreal, True)
        fake_tensor = self.get_target_tensor(Dreal, False)

        if self.which_D == 'S':
            prediction_fake = Dfake
            prediction_real = real_tensor if self.loss_mode in ['vanilla'] else fake_tensor
        elif self.which_D == 'Ra':
            prediction_fake = Dfake - torch.mean(Dreal)
            prediction_real = Dreal - torch.mean(Dfake)
        else:
            raise NotImplementedError('which_D name [%s] is not recognized' % self.which_D)

        if self.loss_mode in ['lsgan', 'nsgan']:
            loss_fake = self.loss(prediction_fake, real_tensor)
            loss_real = self.loss(prediction_real, fake_tensor)
        elif self.loss_mode == 'vanilla':
            loss_fake = -self.loss(prediction_fake, fake_tensor)
            loss_real = -self.loss(prediction_real, real_tensor)
        elif self.loss_mode in ['wgan', 'hinge'] and self.which_D == 'S':
            loss_fake = -prediction_fake.mean()
            loss_real = prediction_real.mean()
        elif self.loss_mode == 'hinge' and self.which_D == 'Ra':
            loss_fake = nn.ReLU()(1.0 - prediction_fake).mean()
            loss_real = nn.ReLU()(1.0 + prediction_real).mean()
        elif self.loss_mode == 'rsgan':
            loss_fake = self.loss(Dfake - Dreal, real_tensor)
            loss_real = 0.
        else:
            raise NotImplementedError('loss_mode name [%s] is not recognized' % self.loss_mode)

        return loss_fake, loss_real

    def D_loss(self, Dfake, Dreal):
        real_tensor = self.get_target_tensor(Dreal, True)
        fake_tensor = self.get_target_tensor(Dreal, False)

        if self.which_D == 'S':
            prediction_fake = Dfake
            prediction_real = Dreal
        elif self.which_D == 'Ra':
            prediction_fake = Dfake - torch.mean(Dreal)
            prediction_real = Dreal - torch.mean(Dfake)
        else:
            raise NotImplementedError('which_D name [%s] is not recognized' % self.which_D)

        if self.loss_mode in ['lsgan', 'nsgan', 'vanilla']:
            loss_fake = self.loss(prediction_fake, fake_tensor)
            loss_real = self.loss(prediction_real, real_tensor)
        elif self.loss_mode == 'wgan':
            loss_fake = prediction_fake.mean()
            loss_real = -prediction_real.mean()
        elif self.loss_mode == 'hinge':
            loss_fake = nn.ReLU()(1.0 + prediction_fake).mean()
            loss_real = nn.ReLU()(1.0 - prediction_real).mean()
        elif self.loss_mode == 'rsgan':
            loss_fake = 0.
            loss_real = self.loss(Dreal - Dfake, real_tensor)
        else:
            raise NotImplementedError('loss_mode name [%s] is not recognized' % self.loss_mode)

        return loss_fake, loss_real

    def __call__(self, Dfake, Dreal):
        """Calculate loss given Discriminator's output and grount truth labels.
        Parameters:
            prediction (tensor) - - tpyically the prediction output from a discriminator
            target_is_real (bool) - - if the ground truth label is for real images or fake images
        Returns:
            the calculated loss.
        """
        if self.which_net == 'G':
            loss_fake, loss_real = self.G_loss(Dfake, Dreal)
            return loss_fake, loss_real
        elif self.which_net == 'D':
            loss_fake, loss_real = self.D_loss(Dfake, Dreal)
            return loss_fake, loss_real
        else:
            raise NotImplementedError('which_net name [%s] is not recognized' % self.which_net)


def cal_gradient_penalty(netD, real_data, fake_data, device, type='mixed', constant=1.0, lambda_gp=10.0):
    """Calculate the gradient penalty loss, used in WGAN-GP paper https://arxiv.org/abs/1704.00028

    Arguments:
        netD (network)              -- discriminator network
        real_data (tensor array)    -- real images
        fake_data (tensor array)    -- generated images from the generator
        device (str)                -- GPU / CPU:
            from torch.device('cuda:{}'.format(self.gpu_ids[0])) if self.gpu_ids else torch.device('cpu')
        type (str)                  -- if we mix real and fake data or not [real | fake | mixed].
        constant (float)            -- the constant used in formula ( | |gradient||_2 - constant)^2
        lambda_gp (float)           -- weight for this loss

    Returns the gradient penalty loss
    """
    if lambda_gp > 0.0:
        if type == 'real':  # either use real images, fake images, or a linear interpolation of two.
            interpolatesv = real_data
        elif type == 'fake':
            interpolatesv = fake_data
        elif type == 'mixed':
            alpha = torch.rand(real_data.shape[0], 1)
            alpha = alpha.expand(real_data.shape[0], real_data.nelement() // real_data.shape[0]).contiguous().view(
                *real_data.shape)
            alpha = alpha.to(device)
            interpolatesv = alpha * real_data + ((1 - alpha) * fake_data)
        else:
            raise NotImplementedError('{} not implemented'.format(type))
        interpolatesv.requires_grad_(True)
        disc_interpolates = netD({'data': interpolatesv})
        gradients = torch.autograd.grad(outputs=disc_interpolates, inputs=interpolatesv,
                                        grad_outputs=torch.ones(disc_interpolates.size()).to(device),
                                        create_graph=True, retain_graph=True, only_inputs=True)
        gradients = gradients[0].view(real_data.size(0), -1)  # flat the data
        gradient_penalty = (((gradients + 1e-16).norm(2, dim=1) - constant) ** 2).mean() * lambda_gp  # added eps
        return gradient_penalty, gradients
    else:
        return 0.0, None

<filename>models/networks/loss.py
import torch
from torch import nn


class GANLoss(nn.Module):
    """Define different GAN Discriminator's objectives.

    The GANLoss class abstracts away the need to create the target label tensor
    that has the same size as the input.
    """

    def __init__(self, loss_mode, which_net, which_D, target_real_label=1.0, target_fake_label=0.0):
        """ Initialize the GAN's Discriminator Loss class.

        Parameters:
            loss_mode (str) - - the type of GAN objective. It currently supports vanilla, lsgan, and wgangp.
            target_real_label (bool) - - label for a real image
            target_fake_label (bool) - - label of a fake image

        Note: Do not use sigmoid as the last layer of Discriminator.
        LSGAN needs no sigmoid. vanilla GANs will handle it with BCEWithLogitsLoss.
        """
        super(GANLoss, self).__init__()
        self.register_buffer('real_label', torch.tensor(target_real_label))
        self.register_buffer('fake_label', torch.tensor(target_fake_label))
        self.loss_mode = loss_mode
        self.which_net = which_net
        self.which_D = which_D

        if loss_mode == 'lsgan':
            self.loss = nn.MSELoss()
        elif loss_mode in ['vanilla', 'nsgan', 'rsgan']:
            self.loss = nn.BCEWithLogitsLoss()
        elif loss_mode in ['wgan', 'hinge']:
            self.loss = None
        else:
            raise NotImplementedError('gan mode %s not implemented' % loss_mode)

    def get_target_tensor(self, prediction, target_is_real):
        """Create label tensors with the same size as the input.
        Parameters:
            prediction (tensor) - - tpyically the prediction from a discriminator
            target_is_real (bool) - - if the ground truth label is for real images or fake images
        Returns:
            A label tensor filled with ground truth label, and with the size of the input
        """
        if target_is_real:
            target_tensor = self.real_label
        else:
            target_tensor = self.fake_label
        return target_tensor.expand_as(prediction)

    def G_loss(self, Dfake, Dreal):
        real_tensor = self.get_target_tensor(Dreal, True)
        fake_tensor = self.get_target_tensor(Dreal, False)

        if self.which_D == 'S':
            prediction_fake = Dfake
            prediction_real = real_tensor if self.loss_mode in ['vanilla'] else fake_tensor
        elif self.which_D == 'Ra':
            prediction_fake = Dfake - torch.mean(Dreal)
            prediction_real = Dreal - torch.mean(Dfake)
        else:
            raise NotImplementedError('which_D name [%s] is not recognized' % self.which_D)

        if self.loss_mode in ['lsgan', 'nsgan']:
            loss_fake = self.loss(prediction_fake, real_tensor)
            loss_real = self.loss(prediction_real, fake_tensor)
        elif self.loss_mode == 'vanilla':
            loss_fake = -self.loss(prediction_fake, fake_tensor)
            loss_real = -self.loss(prediction_real, real_tensor)
        elif self.loss_mode in ['wgan', 'hinge'] and self.which_D == 'S':
            loss_fake = -prediction_fake.mean()
            loss_real = prediction_real.mean()
        elif self.loss_mode == 'hinge' and self.which_D == 'Ra':
            loss_fake = nn.ReLU()(1.0 - prediction_fake).mean()
            loss_real = nn.ReLU()(1.0 + prediction_real).mean()
        elif self.loss_mode == 'rsgan':
            loss_fake = self.loss(Dfake - Dreal, real_tensor)
            loss_real = 0.
        else:
            raise NotImplementedError('loss_mode name [%s] is not recognized' % self.loss_mode)

        return loss_fake, loss_real

    def D_loss(self, Dfake, Dreal):
        real_tensor = self.get_target_tensor(Dreal, True)
        fake_tensor = self.get_target_tensor(Dreal, False)

        if self.which_D == 'S':
            prediction_fake = Dfake
            prediction_real = Dreal
        elif self.which_D == 'Ra':
            prediction_fake = Dfake - torch.mean(Dreal)
            prediction_real = Dreal - torch.mean(Dfake)
        else:
            raise NotImplementedError('which_D name [%s] is not recognized' % self.which_D)

        if self.loss_mode in ['lsgan', 'nsgan', 'vanilla']:
            loss_fake = self.loss(prediction_fake, fake_tensor)
            loss_real = self.loss(prediction_real, real_tensor)
        elif self.loss_mode == 'wgan':
            loss_fake = prediction_fake.mean()
            loss_real = -prediction_real.mean()
        elif self.loss_mode == 'hinge':
            loss_fake = nn.ReLU()(1.0 + prediction_fake).mean()
            loss_real = nn.ReLU()(1.0 - prediction_real).mean()
        elif self.loss_mode == 'rsgan':
            loss_fake = 0.
            loss_real = self.loss(Dreal - Dfake, real_tensor)
        else:
            raise NotImplementedError('loss_mode name [%s] is not recognized' % self.loss_mode)

        return loss_fake, loss_real

    def __call__(self, Dfake, Dreal):
        """Calculate loss given Discriminator's output and grount truth labels.
        Parameters:
            prediction (tensor) - - tpyically the prediction output from a discriminator
            target_is_real (bool) - - if the ground truth label is for real images or fake images
        Returns:
            the calculated loss.
        """
        if self.which_net == 'G':
            loss_fake, loss_real = self.G_loss(Dfake, Dreal)
            return loss_fake, loss_real
        elif self.which_net == 'D':
            loss_fake, loss_real = self.D_loss(Dfake, Dreal)
            return loss_fake, loss_real
        else:
            raise NotImplementedError('which_net name [%s] is not recognized' % self.which_net)


def cal_gradient_penalty(netD, real_data, fake_data, device, type='mixed', constant=1.0, lambda_gp=10.0):
    """Calculate the gradient penalty loss, used in WGAN-GP paper https://arxiv.org/abs/1704.00028

    Arguments:
        netD (network)              -- discriminator network
        real_data (tensor array)    -- real images
        fake_data (tensor array)    -- generated images from the generator
        device (str)                -- GPU / CPU:
            from torch.device('cuda:{}'.format(self.gpu_ids[0])) if self.gpu_ids else torch.device('cpu')
        type (str)                  -- if we mix real and fake data or not [real | fake | mixed].
        constant (float)            -- the constant used in formula ( | |gradient||_2 - constant)^2
        lambda_gp (float)           -- weight for this loss

    Returns the gradient penalty loss
    """
    if lambda_gp > 0.0:
        if type == 'real':  # either use real images, fake images, or a linear interpolation of two.
            interpolatesv = real_data
        elif type == 'fake':
            interpolatesv = fake_data
        elif type == 'mixed':
            alpha = torch.rand(real_data.shape[0], 1)
            alpha = alpha.expand(real_data.shape[0], real_data.nelement() // real_data.shape[0]).contiguous().view(
                *real_data.shape)
            alpha = alpha.to(device)
            interpolatesv = alpha * real_data + ((1 - alpha) * fake_data)
        else:
            raise NotImplementedError('{} not implemented'.format(type))
        interpolatesv.requires_grad_(True)
        disc_interpolates = netD({'data': interpolatesv})
        gradients = torch.autograd.grad(outputs=disc_interpolates, inputs=interpolatesv,
                                        grad_outputs=torch.ones(disc_interpolates.size()).to(device),
                                        create_graph=True, retain_graph=True, only_inputs=True)
        gradients = gradients[0].view(real_data.size(0), -1)  # flat the data
        gradient_penalty = (((gradients + 1e-16).norm(2, dim=1) - constant) ** 2).mean() * lambda_gp  # added eps
        return gradient_penalty, gradients
    else:
        return 0.0, None

Define different GAN Discriminator's objectives.      The GANLoss class abstracts away the need to create the target label tensor     that has the same size as the input. Initialize the GAN's Discriminator Loss class.          Parameters:             loss_mode (str) - - the type of GAN objective. It currently supports vanilla, lsgan, and wgangp.             target_real_label (bool) - - label for a real image             target_fake_label (bool) - - label of a fake image          Note: Do not use sigmoid as the last layer of Discriminator.         LSGAN needs no sigmoid. vanilla GANs will handle it with BCEWithLogitsLoss. Create label tensors with the same size as the input.         Parameters:             prediction (tensor) - - tpyically the prediction from a discriminator             target_is_real (bool) - - if the ground truth label is for real images or fake images         Returns:             A label tensor filled with ground truth label, and with the size of the input Calculate loss given Discriminator's output and grount truth labels.         Parameters:             prediction (tensor) - - tpyically the prediction output from a discriminator             target_is_real (bool) - - if the ground truth label is for real images or fake images         Returns:             the calculated loss. Calculate the gradient penalty loss, used in WGAN-GP paper https://arxiv.org/abs/1704.00028      Arguments:         netD (network)              -- discriminator network         real_data (tensor array)    -- real images         fake_data (tensor array)    -- generated images from the generator         device (str)                -- GPU / CPU:             from torch.device('cuda:{}'.format(self.gpu_ids[0])) if self.gpu_ids else torch.device('cpu')         type (str)                  -- if we mix real and fake data or not [real | fake | mixed].         constant (float)            -- the constant used in formula ( | |gradient||_2 - constant)^2         lambda_gp (float)           -- weight for this loss      Returns the gradient penalty loss # either use real images, fake images, or a linear interpolation of two. # flat the data # added eps