<gh_stars>0
import copy
import os.path

import numpy as np
import torch
import torch.nn.functional as F
import pytorch_lightning as pl

from main import instantiate_from_config

from taming.modules.diffusionmodules.model import Encoder, Decoder
from taming.modules.vqvae.quantize import VectorQuantizer2 as VectorQuantizer
from taming.modules.vqvae.quantize import GumbelQuantize
from taming.modules.vqvae.quantize import EMAVectorQuantizer
from torch_utils import training_stats


class VQModel(pl.LightningModule):
    def __init__(self,
                 ddconfig,
                 lossconfig,
                 n_embed,
                 embed_dim,
                 ckpt_path=None,
                 ignore_keys=[],
                 image_key="image",
                 disc_image_key="disc_image",
                 colorize_nlabels=None,
                 monitor=None,
                 remap=None,
                 sane_index_shape=False,  # tell vector quantizer to return indices as bhw
                 ada_target=0.6,
                 ada_interval=4,
                 ada_kimg=500,
                 ):
        super().__init__()
        self.image_key = image_key
        self.disc_image_key = disc_image_key
        self.calc_adv_loss = False
        self.update_ada = False
        self.ada_target = ada_target
        self.ada_interval = ada_interval
        self.ada_kimg = ada_kimg
        self.ada_stats_regex = 'Loss/D/sign/logits_real'
        self.ada_stats = training_stats.Collector(regex=self.ada_stats_regex) \
            if ada_target is not None else None
        self.encoder = Encoder(**ddconfig)
        self.decoder = Decoder(**ddconfig)
        self.loss = instantiate_from_config(lossconfig)
        self.quantize = VectorQuantizer(n_embed, embed_dim, beta=0.25,
                                        remap=remap, sane_index_shape=sane_index_shape)
        self.quant_conv = torch.nn.Conv2d(ddconfig["z_channels"], embed_dim, 1)
        self.post_quant_conv = torch.nn.Conv2d(embed_dim, ddconfig["z_channels"], 1)
        if ckpt_path is not None:
            self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys)
        if colorize_nlabels is not None:
            assert type(colorize_nlabels) == int
            self.register_buffer("colorize", torch.randn(3, colorize_nlabels, 1, 1))
        if monitor is not None:
            self.monitor = monitor

        random_latent = torch.randn((36, *self.decoder.z_shape[1:]))
        self.register_buffer('zs', random_latent)

    def init_from_ckpt(self, path, ignore_keys=list()):
        sd = torch.load(path, map_location="cpu")["state_dict"]
        keys = list(sd.keys())
        for k in keys:
            for ik in ignore_keys:
                if k.startswith(ik):
                    print("Deleting key {} from state_dict.".format(k))
                    del sd[k]
        self.load_state_dict(sd, strict=False)
        print(f"Restored from {path}")

    def encode(self, x):
        h = self.encoder(x)
        h = self.quant_conv(h)
        quant, emb_loss, info = self.quantize(h)
        return quant, emb_loss, info

    def decode(self, quant):
        quant = self.post_quant_conv(quant)
        dec = self.decoder(quant)
        return dec

    def decode_code(self, code_b):
        quant_b = self.quantize.embed_code(code_b)
        dec = self.decode(quant_b)
        return dec

    def forward(self, input, return_quant=False):
        quant, diff, _ = self.encode(input)
        dec = self.decode(quant)
        if not return_quant:
            return dec, diff
        return dec, quant, diff

    def forward_with_latent(self, z):
        fake = self.decoder(z)
        return fake

    def get_input(self, batch, k):
        x = batch[k]
        if len(x.shape) == 3:
            x = x[..., None]
        x = x.permute(0, 3, 1, 2).to(memory_format=torch.contiguous_format)
        return x.float()

    def on_train_batch_start(self, batch, batch_idx, dataloader_idx):
        if self.global_step >= self.loss.discriminator_iter_start:
            self.calc_adv_loss = True
            self.update_ada = True
            self.train_dataloader().dataset.data.prepare_disc_data(True)

    def training_step(self, batch, batch_idx, optimizer_idx):
        if optimizer_idx == 0:
            x = self.get_input(batch, self.image_key)
            xrec, quant, qloss = self(x, return_quant=True)
            quant_var, quant_mean = torch.var_mean(quant)

            if self.calc_adv_loss:
                random_z = self.random_latent(quant_mean, quant_var, quant.shape)
                fake = self.forward_with_latent(random_z)
            else:
                fake = None
            self.real, self.reconstruction, self.fake = x, xrec, fake
            self.quant_var, self.quant_mean, self.qloss = quant_var, quant_mean, qloss

        if optimizer_idx == 0:
            # autoencode
            aeloss, log_dict_ae = \
                self.loss(self.qloss, self.quant_var, self.quant_mean, self.real, self.reconstruction, self.fake,
                          optimizer_idx, self.global_step, last_layer=self.get_last_layer(), split="train")
            self.log_dict(log_dict_ae, prog_bar=False, logger=True, on_step=True, on_epoch=False)
            return aeloss

        if optimizer_idx == 1:
            # discriminator
            disc_x = self.get_input(batch, self.disc_image_key)
            reconstruction = self.reconstruction.detach()
            fake = self.fake.detach() if self.fake is not None else None
            discloss, log_dict_disc = \
                self.loss(None, None, None, disc_x, reconstruction, fake,
                          optimizer_idx, self.global_step, split="train")
            if self.calc_adv_loss:
                log_dict_disc.update({'ada/aug_prob': self.get_disc_aug_p()})
            self.log_dict(log_dict_disc, prog_bar=False, logger=True, on_step=True, on_epoch=False)
            if self.calc_adv_loss:
                training_stats.report(self.ada_stats_regex, 1 - log_dict_disc['train_adversarial_D/disc_loss_real'])
            return discloss

    def validation_step(self, batch, batch_idx):
        x = self.get_input(batch, self.image_key)
        xrec, quant, qloss = self(x, return_quant=True)
        quant_var, quant_mean = torch.var_mean(quant)

        if self.global_step >= self.loss.discriminator_iter_start:
            random_z = self.random_latent(quant_mean, quant_var, quant.shape)
            fake = self.forward_with_latent(random_z)
        else:
            fake = None

        aeloss, log_dict_ae = \
            self.loss(qloss, quant_var, quant_mean, x, xrec, fake,
                      0, self.global_step, last_layer=self.get_last_layer(), split="val")
        discloss, log_dict_disc = self.loss(None, None, None, x, xrec, fake, 1, self.global_step, split="val")
        self.log_dict(log_dict_ae, prog_bar=False, logger=True, on_step=False, on_epoch=True)
        self.log_dict(log_dict_disc, prog_bar=False, logger=True, on_step=False, on_epoch=True)

        return {
            'epoch': self.current_epoch,
            'sup': log_dict_ae['val_supervised/nll_loss'],
            'adv_rec': log_dict_ae.get('val_adversarial/G/g_rec_loss', 0.),
            'adv_fake': log_dict_ae.get('val_adversarial/G/g_fake_loss', 0.),
            'total': log_dict_ae['val_total/total_loss'] - log_dict_ae['val_supervised/quant_loss'],
        }

    def test_step(self, batch, batch_idx):
        x = self.get_input(batch, self.image_key)
        xrec, qloss = self(x)

        aeloss, log_dict_ae = \
            self.loss(qloss, None, None, x, xrec, None,
                      0, self.global_step, last_layer=self.get_last_layer(), split="test")
        discloss, log_dict_disc = self.loss(None, None, None, x, xrec, None, 1, self.global_step, split="test")
        if self.global_step != 0:
            self.log_dict(log_dict_ae, prog_bar=False, logger=True, on_step=False, on_epoch=True)
            self.log_dict(log_dict_disc, prog_bar=False, logger=True, on_step=False, on_epoch=True)

        return {
            'epoch': self.current_epoch,
            'sup': log_dict_ae['test_supervised/nll_loss'],
            'total': log_dict_ae['test_total/total_loss'] - log_dict_ae['test_supervised/quant_loss'],
        }

    def on_after_backward(self):
        if self.global_step % self.ada_interval == 0 and self.calc_adv_loss and self.update_ada \
                and self.ada_stats is not None:
            self.ada_stats.update()
            sign = np.sign(self.ada_stats[self.ada_stats_regex] - self.ada_target)
            value = self.train_dataloader().batch_size * self.trainer.accumulate_grad_batches * self.ada_interval / (
                        self.ada_kimg * 1000)
            adjust = sign * value
            self.adjust_disc_aug_p(adjust)
            self.update_ada = False

    def configure_optimizers(self):
        lr = self.learning_rate
        opt_ae = torch.optim.Adam(list(self.encoder.parameters())+
                                  list(self.decoder.parameters())+
                                  list(self.quantize.parameters())+
                                  list(self.quant_conv.parameters())+
                                  list(self.post_quant_conv.parameters()),
                                  lr=lr, betas=(0.5, 0.9))
        opt_disc = torch.optim.Adam(self.loss.parameters(), lr=lr, betas=(0.5, 0.9))
        return [opt_ae, opt_disc], []

    @staticmethod
    def random_latent(mean, var, shape):
        return torch.normal(mean.expand(shape), var.expand(shape))

    def get_last_layer(self):
        return self.decoder.conv_out.weight

    def log_images(self, batch, log_real=False, log_fake=False, **kwargs):
        x = self.get_input(batch, self.image_key)
        x = x.to(self.device)
        xrec, quant, _ = self(x, True)
        if x.shape[1] > 3:
            # colorize with random projection
            assert xrec.shape[1] > 3
            x = self.to_rgb(x)
            xrec = self.to_rgb(xrec)

        log = {
            'inputs': x,
            'reconstructions': xrec,
        }

        if log_real:
            real = self.get_input(batch, self.disc_image_key)
            if real.shape[1] > 3:
                real = self.to_rgb(real)
            log['real'] = real
        if log_fake:
            fake = self.forward_with_latent(self.zs)
            if fake.shape[1] > 3:
                fake = self.to_rgb(fake)
            log['fake'] = fake

        return log, quant

    def to_rgb(self, x):
        assert self.image_key == "segmentation"
        if not hasattr(self, "colorize"):
            self.register_buffer("colorize", torch.randn(3, x.shape[1], 1, 1).to(x))
        x = F.conv2d(x, weight=self.colorize)
        x = 2.*(x-x.min())/(x.max()-x.min()) - 1.
        return x

    def adjust_disc_aug_p(self, adjust):
        self.train_dataloader().dataset.data.adjust_disc_aug_p(adjust)

    def get_disc_aug_p(self):
        return torch.tensor(self.train_dataloader().dataset.data.disc_aug_p())


class VQSegmentationModel(VQModel):
    def __init__(self, n_labels, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.register_buffer("colorize", torch.randn(3, n_labels, 1, 1))

    def configure_optimizers(self):
        lr = self.learning_rate
        opt_ae = torch.optim.Adam(list(self.encoder.parameters())+
                                  list(self.decoder.parameters())+
                                  list(self.quantize.parameters())+
                                  list(self.quant_conv.parameters())+
                                  list(self.post_quant_conv.parameters()),
                                  lr=lr, betas=(0.5, 0.9))
        return opt_ae

    def training_step(self, batch, batch_idx):
        x = self.get_input(batch, self.image_key)
        xrec, qloss = self(x)
        aeloss, log_dict_ae = self.loss(qloss, x, xrec, split="train")
        self.log_dict(log_dict_ae, prog_bar=False, logger=True, on_step=True, on_epoch=True)
        return aeloss

    def validation_step(self, batch, batch_idx):
        x = self.get_input(batch, self.image_key)
        xrec, qloss = self(x)
        aeloss, log_dict_ae = self.loss(qloss, x, xrec, split="val")
        self.log_dict(log_dict_ae, prog_bar=False, logger=True, on_step=True, on_epoch=True)
        total_loss = log_dict_ae["val/total_loss"]
        self.log("val/total_loss", total_loss,
                 prog_bar=True, logger=True, on_step=True, on_epoch=True, sync_dist=True)
        return aeloss

    @torch.no_grad()
    def log_images(self, batch, **kwargs):
        log = dict()
        x = self.get_input(batch, self.image_key)
        x = x.to(self.device)
        xrec, _ = self(x)
        if x.shape[1] > 3:
            # colorize with random projection
            assert xrec.shape[1] > 3
            # convert logits to indices
            xrec = torch.argmax(xrec, dim=1, keepdim=True)
            xrec = F.one_hot(xrec, num_classes=x.shape[1])
            xrec = xrec.squeeze(1).permute(0, 3, 1, 2).float()
            x = self.to_rgb(x)
            xrec = self.to_rgb(xrec)
        log["inputs"] = x
        log["reconstructions"] = xrec
        return log


class VQNoDiscModel(VQModel):
    def __init__(self,
                 ddconfig,
                 lossconfig,
                 n_embed,
                 embed_dim,
                 ckpt_path=None,
                 ignore_keys=[],
                 image_key="image",
                 colorize_nlabels=None
                 ):
        super().__init__(ddconfig=ddconfig, lossconfig=lossconfig, n_embed=n_embed, embed_dim=embed_dim,
                         ckpt_path=ckpt_path, ignore_keys=ignore_keys, image_key=image_key,
                         colorize_nlabels=colorize_nlabels)

    def training_step(self, batch, batch_idx):
        x = self.get_input(batch, self.image_key)
        xrec, qloss = self(x)
        # autoencode
        aeloss, log_dict_ae = self.loss(qloss, x, xrec, self.global_step, split="train")
        output = pl.TrainResult(minimize=aeloss)
        output.log("train/aeloss", aeloss,
                   prog_bar=True, logger=True, on_step=True, on_epoch=True)
        output.log_dict(log_dict_ae, prog_bar=False, logger=True, on_step=True, on_epoch=True)
        return output

    def validation_step(self, batch, batch_idx):
        x = self.get_input(batch, self.image_key)
        xrec, qloss = self(x)
        aeloss, log_dict_ae = self.loss(qloss, x, xrec, self.global_step, split="val")
        rec_loss = log_dict_ae["val/rec_loss"]
        output = pl.EvalResult(checkpoint_on=rec_loss)
        output.log("val/rec_loss", rec_loss,
                   prog_bar=True, logger=True, on_step=True, on_epoch=True)
        output.log("val/aeloss", aeloss,
                   prog_bar=True, logger=True, on_step=True, on_epoch=True)
        output.log_dict(log_dict_ae)

        return output

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(list(self.encoder.parameters())+
                                  list(self.decoder.parameters())+
                                  list(self.quantize.parameters())+
                                  list(self.quant_conv.parameters())+
                                  list(self.post_quant_conv.parameters()),
                                  lr=self.learning_rate, betas=(0.5, 0.9))
        return optimizer


class GumbelVQ(VQModel):
    def __init__(self,
                 ddconfig,
                 lossconfig,
                 n_embed,
                 embed_dim,
                 temperature_scheduler_config,
                 ckpt_path=None,
                 ignore_keys=[],
                 image_key="image",
                 colorize_nlabels=None,
                 monitor=None,
                 kl_weight=1e-8,
                 remap=None,
                 ):

        z_channels = ddconfig["z_channels"]
        super().__init__(ddconfig,
                         lossconfig,
                         n_embed,
                         embed_dim,
                         ckpt_path=None,
                         ignore_keys=ignore_keys,
                         image_key=image_key,
                         colorize_nlabels=colorize_nlabels,
                         monitor=monitor,
                         )

        self.loss.n_classes = n_embed
        self.vocab_size = n_embed

        self.quantize = GumbelQuantize(z_channels, embed_dim,
                                       n_embed=n_embed,
                                       kl_weight=kl_weight, temp_init=1.0,
                                       remap=remap)

        self.temperature_scheduler = instantiate_from_config(temperature_scheduler_config)   # annealing of temp

        if ckpt_path is not None:
            self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys)

    def temperature_scheduling(self):
        self.quantize.temperature = self.temperature_scheduler(self.global_step)

    def encode_to_prequant(self, x):
        h = self.encoder(x)
        h = self.quant_conv(h)
        return h

    def decode_code(self, code_b):
        raise NotImplementedError

    def training_step(self, batch, batch_idx, optimizer_idx):
        self.temperature_scheduling()
        x = self.get_input(batch, self.image_key)
        xrec, qloss = self(x)

        if optimizer_idx == 0:
            # autoencode
            aeloss, log_dict_ae = self.loss(qloss, x, xrec, optimizer_idx, self.global_step,
                                            last_layer=self.get_last_layer(), split="train")

            self.log_dict(log_dict_ae, prog_bar=False, logger=True, on_step=True, on_epoch=True)
            self.log("temperature", self.quantize.temperature, prog_bar=False, logger=True, on_step=True, on_epoch=True)
            return aeloss

        if optimizer_idx == 1:
            # discriminator
            discloss, log_dict_disc = self.loss(qloss, x, xrec, optimizer_idx, self.global_step,
                                            last_layer=self.get_last_layer(), split="train")
            self.log_dict(log_dict_disc, prog_bar=False, logger=True, on_step=True, on_epoch=True)
            return discloss

    def validation_step(self, batch, batch_idx):
        x = self.get_input(batch, self.image_key)
        xrec, qloss = self(x, return_pred_indices=True)
        aeloss, log_dict_ae = self.loss(qloss, x, xrec, 0, self.global_step,
                                        last_layer=self.get_last_layer(), split="val")

        discloss, log_dict_disc = self.loss(qloss, x, xrec, 1, self.global_step,
                                            last_layer=self.get_last_layer(), split="val")
        rec_loss = log_dict_ae["val/rec_loss"]
        self.log("val/rec_loss", rec_loss,
                 prog_bar=True, logger=True, on_step=False, on_epoch=True, sync_dist=True)
        self.log("val/aeloss", aeloss,
                 prog_bar=True, logger=True, on_step=False, on_epoch=True, sync_dist=True)
        self.log_dict(log_dict_ae)
        self.log_dict(log_dict_disc)
        return self.log_dict

    def log_images(self, batch, **kwargs):
        log = dict()
        x = self.get_input(batch, self.image_key)
        x = x.to(self.device)
        # encode
        h = self.encoder(x)
        h = self.quant_conv(h)
        quant, _, _ = self.quantize(h)
        # decode
        x_rec = self.decode(quant)
        log["inputs"] = x
        log["reconstructions"] = x_rec
        return log


class EMAVQ(VQModel):
    def __init__(self,
                 ddconfig,
                 lossconfig,
                 n_embed,
                 embed_dim,
                 ckpt_path=None,
                 ignore_keys=[],
                 image_key="image",
                 colorize_nlabels=None,
                 monitor=None,
                 remap=None,
                 sane_index_shape=False,  # tell vector quantizer to return indices as bhw
                 ):
        super().__init__(ddconfig,
                         lossconfig,
                         n_embed,
                         embed_dim,
                         ckpt_path=None,
                         ignore_keys=ignore_keys,
                         image_key=image_key,
                         colorize_nlabels=colorize_nlabels,
                         monitor=monitor,
                         )
        self.quantize = EMAVectorQuantizer(n_embed=n_embed,
                                           embedding_dim=embed_dim,
                                           beta=0.25,
                                           remap=remap)

    def configure_optimizers(self):
        lr = self.learning_rate
        # Remove self.quantize from parameter list since it is updated via EMA
        opt_ae = torch.optim.Adam(list(self.encoder.parameters())+
                                  list(self.decoder.parameters())+
                                  list(self.quant_conv.parameters())+
                                  list(self.post_quant_conv.parameters()),
                                  lr=lr, betas=(0.5, 0.9))
        opt_disc = torch.optim.Adam(self.loss.discriminator.parameters(),
                                    lr=lr, betas=(0.5, 0.9))
        return [opt_ae, opt_disc], []

import copy
import os.path

import numpy as np
import torch
import torch.nn.functional as F
import pytorch_lightning as pl

from main import instantiate_from_config

from taming.modules.diffusionmodules.model import Encoder, Decoder
from taming.modules.vqvae.quantize import VectorQuantizer2 as VectorQuantizer
from taming.modules.vqvae.quantize import GumbelQuantize
from taming.modules.vqvae.quantize import EMAVectorQuantizer
from torch_utils import training_stats


class VQModel(pl.LightningModule):
    def __init__(self,
                 ddconfig,
                 lossconfig,
                 n_embed,
                 embed_dim,
                 ckpt_path=None,
                 ignore_keys=[],
                 image_key="image",
                 disc_image_key="disc_image",
                 colorize_nlabels=None,
                 monitor=None,
                 remap=None,
                 sane_index_shape=False,  # tell vector quantizer to return indices as bhw
                 ada_target=0.6,
                 ada_interval=4,
                 ada_kimg=500,
                 ):
        super().__init__()
        self.image_key = image_key
        self.disc_image_key = disc_image_key
        self.calc_adv_loss = False
        self.update_ada = False
        self.ada_target = ada_target
        self.ada_interval = ada_interval
        self.ada_kimg = ada_kimg
        self.ada_stats_regex = 'Loss/D/sign/logits_real'
        self.ada_stats = training_stats.Collector(regex=self.ada_stats_regex) \
            if ada_target is not None else None
        self.encoder = Encoder(**ddconfig)
        self.decoder = Decoder(**ddconfig)
        self.loss = instantiate_from_config(lossconfig)
        self.quantize = VectorQuantizer(n_embed, embed_dim, beta=0.25,
                                        remap=remap, sane_index_shape=sane_index_shape)
        self.quant_conv = torch.nn.Conv2d(ddconfig["z_channels"], embed_dim, 1)
        self.post_quant_conv = torch.nn.Conv2d(embed_dim, ddconfig["z_channels"], 1)
        if ckpt_path is not None:
            self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys)
        if colorize_nlabels is not None:
            assert type(colorize_nlabels) == int
            self.register_buffer("colorize", torch.randn(3, colorize_nlabels, 1, 1))
        if monitor is not None:
            self.monitor = monitor

        random_latent = torch.randn((36, *self.decoder.z_shape[1:]))
        self.register_buffer('zs', random_latent)

    def init_from_ckpt(self, path, ignore_keys=list()):
        sd = torch.load(path, map_location="cpu")["state_dict"]
        keys = list(sd.keys())
        for k in keys:
            for ik in ignore_keys:
                if k.startswith(ik):
                    print("Deleting key {} from state_dict.".format(k))
                    del sd[k]
        self.load_state_dict(sd, strict=False)
        print(f"Restored from {path}")

    def encode(self, x):
        h = self.encoder(x)
        h = self.quant_conv(h)
        quant, emb_loss, info = self.quantize(h)
        return quant, emb_loss, info

    def decode(self, quant):
        quant = self.post_quant_conv(quant)
        dec = self.decoder(quant)
        return dec

    def decode_code(self, code_b):
        quant_b = self.quantize.embed_code(code_b)
        dec = self.decode(quant_b)
        return dec

    def forward(self, input, return_quant=False):
        quant, diff, _ = self.encode(input)
        dec = self.decode(quant)
        if not return_quant:
            return dec, diff
        return dec, quant, diff

    def forward_with_latent(self, z):
        fake = self.decoder(z)
        return fake

    def get_input(self, batch, k):
        x = batch[k]
        if len(x.shape) == 3:
            x = x[..., None]
        x = x.permute(0, 3, 1, 2).to(memory_format=torch.contiguous_format)
        return x.float()

    def on_train_batch_start(self, batch, batch_idx, dataloader_idx):
        if self.global_step >= self.loss.discriminator_iter_start:
            self.calc_adv_loss = True
            self.update_ada = True
            self.train_dataloader().dataset.data.prepare_disc_data(True)

    def training_step(self, batch, batch_idx, optimizer_idx):
        if optimizer_idx == 0:
            x = self.get_input(batch, self.image_key)
            xrec, quant, qloss = self(x, return_quant=True)
            quant_var, quant_mean = torch.var_mean(quant)

            if self.calc_adv_loss:
                random_z = self.random_latent(quant_mean, quant_var, quant.shape)
                fake = self.forward_with_latent(random_z)
            else:
                fake = None
            self.real, self.reconstruction, self.fake = x, xrec, fake
            self.quant_var, self.quant_mean, self.qloss = quant_var, quant_mean, qloss

        if optimizer_idx == 0:
            # autoencode
            aeloss, log_dict_ae = \
                self.loss(self.qloss, self.quant_var, self.quant_mean, self.real, self.reconstruction, self.fake,
                          optimizer_idx, self.global_step, last_layer=self.get_last_layer(), split="train")
            self.log_dict(log_dict_ae, prog_bar=False, logger=True, on_step=True, on_epoch=False)
            return aeloss

        if optimizer_idx == 1:
            # discriminator
            disc_x = self.get_input(batch, self.disc_image_key)
            reconstruction = self.reconstruction.detach()
            fake = self.fake.detach() if self.fake is not None else None
            discloss, log_dict_disc = \
                self.loss(None, None, None, disc_x, reconstruction, fake,
                          optimizer_idx, self.global_step, split="train")
            if self.calc_adv_loss:
                log_dict_disc.update({'ada/aug_prob': self.get_disc_aug_p()})
            self.log_dict(log_dict_disc, prog_bar=False, logger=True, on_step=True, on_epoch=False)
            if self.calc_adv_loss:
                training_stats.report(self.ada_stats_regex, 1 - log_dict_disc['train_adversarial_D/disc_loss_real'])
            return discloss

    def validation_step(self, batch, batch_idx):
        x = self.get_input(batch, self.image_key)
        xrec, quant, qloss = self(x, return_quant=True)
        quant_var, quant_mean = torch.var_mean(quant)

        if self.global_step >= self.loss.discriminator_iter_start:
            random_z = self.random_latent(quant_mean, quant_var, quant.shape)
            fake = self.forward_with_latent(random_z)
        else:
            fake = None

        aeloss, log_dict_ae = \
            self.loss(qloss, quant_var, quant_mean, x, xrec, fake,
                      0, self.global_step, last_layer=self.get_last_layer(), split="val")
        discloss, log_dict_disc = self.loss(None, None, None, x, xrec, fake, 1, self.global_step, split="val")
        self.log_dict(log_dict_ae, prog_bar=False, logger=True, on_step=False, on_epoch=True)
        self.log_dict(log_dict_disc, prog_bar=False, logger=True, on_step=False, on_epoch=True)

        return {
            'epoch': self.current_epoch,
            'sup': log_dict_ae['val_supervised/nll_loss'],
            'adv_rec': log_dict_ae.get('val_adversarial/G/g_rec_loss', 0.),
            'adv_fake': log_dict_ae.get('val_adversarial/G/g_fake_loss', 0.),
            'total': log_dict_ae['val_total/total_loss'] - log_dict_ae['val_supervised/quant_loss'],
        }

    def test_step(self, batch, batch_idx):
        x = self.get_input(batch, self.image_key)
        xrec, qloss = self(x)

        aeloss, log_dict_ae = \
            self.loss(qloss, None, None, x, xrec, None,
                      0, self.global_step, last_layer=self.get_last_layer(), split="test")
        discloss, log_dict_disc = self.loss(None, None, None, x, xrec, None, 1, self.global_step, split="test")
        if self.global_step != 0:
            self.log_dict(log_dict_ae, prog_bar=False, logger=True, on_step=False, on_epoch=True)
            self.log_dict(log_dict_disc, prog_bar=False, logger=True, on_step=False, on_epoch=True)

        return {
            'epoch': self.current_epoch,
            'sup': log_dict_ae['test_supervised/nll_loss'],
            'total': log_dict_ae['test_total/total_loss'] - log_dict_ae['test_supervised/quant_loss'],
        }

    def on_after_backward(self):
        if self.global_step % self.ada_interval == 0 and self.calc_adv_loss and self.update_ada \
                and self.ada_stats is not None:
            self.ada_stats.update()
            sign = np.sign(self.ada_stats[self.ada_stats_regex] - self.ada_target)
            value = self.train_dataloader().batch_size * self.trainer.accumulate_grad_batches * self.ada_interval / (
                        self.ada_kimg * 1000)
            adjust = sign * value
            self.adjust_disc_aug_p(adjust)
            self.update_ada = False

    def configure_optimizers(self):
        lr = self.learning_rate
        opt_ae = torch.optim.Adam(list(self.encoder.parameters())+
                                  list(self.decoder.parameters())+
                                  list(self.quantize.parameters())+
                                  list(self.quant_conv.parameters())+
                                  list(self.post_quant_conv.parameters()),
                                  lr=lr, betas=(0.5, 0.9))
        opt_disc = torch.optim.Adam(self.loss.parameters(), lr=lr, betas=(0.5, 0.9))
        return [opt_ae, opt_disc], []

    @staticmethod
    def random_latent(mean, var, shape):
        return torch.normal(mean.expand(shape), var.expand(shape))

    def get_last_layer(self):
        return self.decoder.conv_out.weight

    def log_images(self, batch, log_real=False, log_fake=False, **kwargs):
        x = self.get_input(batch, self.image_key)
        x = x.to(self.device)
        xrec, quant, _ = self(x, True)
        if x.shape[1] > 3:
            # colorize with random projection
            assert xrec.shape[1] > 3
            x = self.to_rgb(x)
            xrec = self.to_rgb(xrec)

        log = {
            'inputs': x,
            'reconstructions': xrec,
        }

        if log_real:
            real = self.get_input(batch, self.disc_image_key)
            if real.shape[1] > 3:
                real = self.to_rgb(real)
            log['real'] = real
        if log_fake:
            fake = self.forward_with_latent(self.zs)
            if fake.shape[1] > 3:
                fake = self.to_rgb(fake)
            log['fake'] = fake

        return log, quant

    def to_rgb(self, x):
        assert self.image_key == "segmentation"
        if not hasattr(self, "colorize"):
            self.register_buffer("colorize", torch.randn(3, x.shape[1], 1, 1).to(x))
        x = F.conv2d(x, weight=self.colorize)
        x = 2.*(x-x.min())/(x.max()-x.min()) - 1.
        return x

    def adjust_disc_aug_p(self, adjust):
        self.train_dataloader().dataset.data.adjust_disc_aug_p(adjust)

    def get_disc_aug_p(self):
        return torch.tensor(self.train_dataloader().dataset.data.disc_aug_p())


class VQSegmentationModel(VQModel):
    def __init__(self, n_labels, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.register_buffer("colorize", torch.randn(3, n_labels, 1, 1))

    def configure_optimizers(self):
        lr = self.learning_rate
        opt_ae = torch.optim.Adam(list(self.encoder.parameters())+
                                  list(self.decoder.parameters())+
                                  list(self.quantize.parameters())+
                                  list(self.quant_conv.parameters())+
                                  list(self.post_quant_conv.parameters()),
                                  lr=lr, betas=(0.5, 0.9))
        return opt_ae

    def training_step(self, batch, batch_idx):
        x = self.get_input(batch, self.image_key)
        xrec, qloss = self(x)
        aeloss, log_dict_ae = self.loss(qloss, x, xrec, split="train")
        self.log_dict(log_dict_ae, prog_bar=False, logger=True, on_step=True, on_epoch=True)
        return aeloss

    def validation_step(self, batch, batch_idx):
        x = self.get_input(batch, self.image_key)
        xrec, qloss = self(x)
        aeloss, log_dict_ae = self.loss(qloss, x, xrec, split="val")
        self.log_dict(log_dict_ae, prog_bar=False, logger=True, on_step=True, on_epoch=True)
        total_loss = log_dict_ae["val/total_loss"]
        self.log("val/total_loss", total_loss,
                 prog_bar=True, logger=True, on_step=True, on_epoch=True, sync_dist=True)
        return aeloss

    @torch.no_grad()
    def log_images(self, batch, **kwargs):
        log = dict()
        x = self.get_input(batch, self.image_key)
        x = x.to(self.device)
        xrec, _ = self(x)
        if x.shape[1] > 3:
            # colorize with random projection
            assert xrec.shape[1] > 3
            # convert logits to indices
            xrec = torch.argmax(xrec, dim=1, keepdim=True)
            xrec = F.one_hot(xrec, num_classes=x.shape[1])
            xrec = xrec.squeeze(1).permute(0, 3, 1, 2).float()
            x = self.to_rgb(x)
            xrec = self.to_rgb(xrec)
        log["inputs"] = x
        log["reconstructions"] = xrec
        return log


class VQNoDiscModel(VQModel):
    def __init__(self,
                 ddconfig,
                 lossconfig,
                 n_embed,
                 embed_dim,
                 ckpt_path=None,
                 ignore_keys=[],
                 image_key="image",
                 colorize_nlabels=None
                 ):
        super().__init__(ddconfig=ddconfig, lossconfig=lossconfig, n_embed=n_embed, embed_dim=embed_dim,
                         ckpt_path=ckpt_path, ignore_keys=ignore_keys, image_key=image_key,
                         colorize_nlabels=colorize_nlabels)

    def training_step(self, batch, batch_idx):
        x = self.get_input(batch, self.image_key)
        xrec, qloss = self(x)
        # autoencode
        aeloss, log_dict_ae = self.loss(qloss, x, xrec, self.global_step, split="train")
        output = pl.TrainResult(minimize=aeloss)
        output.log("train/aeloss", aeloss,
                   prog_bar=True, logger=True, on_step=True, on_epoch=True)
        output.log_dict(log_dict_ae, prog_bar=False, logger=True, on_step=True, on_epoch=True)
        return output

    def validation_step(self, batch, batch_idx):
        x = self.get_input(batch, self.image_key)
        xrec, qloss = self(x)
        aeloss, log_dict_ae = self.loss(qloss, x, xrec, self.global_step, split="val")
        rec_loss = log_dict_ae["val/rec_loss"]
        output = pl.EvalResult(checkpoint_on=rec_loss)
        output.log("val/rec_loss", rec_loss,
                   prog_bar=True, logger=True, on_step=True, on_epoch=True)
        output.log("val/aeloss", aeloss,
                   prog_bar=True, logger=True, on_step=True, on_epoch=True)
        output.log_dict(log_dict_ae)

        return output

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(list(self.encoder.parameters())+
                                  list(self.decoder.parameters())+
                                  list(self.quantize.parameters())+
                                  list(self.quant_conv.parameters())+
                                  list(self.post_quant_conv.parameters()),
                                  lr=self.learning_rate, betas=(0.5, 0.9))
        return optimizer


class GumbelVQ(VQModel):
    def __init__(self,
                 ddconfig,
                 lossconfig,
                 n_embed,
                 embed_dim,
                 temperature_scheduler_config,
                 ckpt_path=None,
                 ignore_keys=[],
                 image_key="image",
                 colorize_nlabels=None,
                 monitor=None,
                 kl_weight=1e-8,
                 remap=None,
                 ):

        z_channels = ddconfig["z_channels"]
        super().__init__(ddconfig,
                         lossconfig,
                         n_embed,
                         embed_dim,
                         ckpt_path=None,
                         ignore_keys=ignore_keys,
                         image_key=image_key,
                         colorize_nlabels=colorize_nlabels,
                         monitor=monitor,
                         )

        self.loss.n_classes = n_embed
        self.vocab_size = n_embed

        self.quantize = GumbelQuantize(z_channels, embed_dim,
                                       n_embed=n_embed,
                                       kl_weight=kl_weight, temp_init=1.0,
                                       remap=remap)

        self.temperature_scheduler = instantiate_from_config(temperature_scheduler_config)   # annealing of temp

        if ckpt_path is not None:
            self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys)

    def temperature_scheduling(self):
        self.quantize.temperature = self.temperature_scheduler(self.global_step)

    def encode_to_prequant(self, x):
        h = self.encoder(x)
        h = self.quant_conv(h)
        return h

    def decode_code(self, code_b):
        raise NotImplementedError

    def training_step(self, batch, batch_idx, optimizer_idx):
        self.temperature_scheduling()
        x = self.get_input(batch, self.image_key)
        xrec, qloss = self(x)

        if optimizer_idx == 0:
            # autoencode
            aeloss, log_dict_ae = self.loss(qloss, x, xrec, optimizer_idx, self.global_step,
                                            last_layer=self.get_last_layer(), split="train")

            self.log_dict(log_dict_ae, prog_bar=False, logger=True, on_step=True, on_epoch=True)
            self.log("temperature", self.quantize.temperature, prog_bar=False, logger=True, on_step=True, on_epoch=True)
            return aeloss

        if optimizer_idx == 1:
            # discriminator
            discloss, log_dict_disc = self.loss(qloss, x, xrec, optimizer_idx, self.global_step,
                                            last_layer=self.get_last_layer(), split="train")
            self.log_dict(log_dict_disc, prog_bar=False, logger=True, on_step=True, on_epoch=True)
            return discloss

    def validation_step(self, batch, batch_idx):
        x = self.get_input(batch, self.image_key)
        xrec, qloss = self(x, return_pred_indices=True)
        aeloss, log_dict_ae = self.loss(qloss, x, xrec, 0, self.global_step,
                                        last_layer=self.get_last_layer(), split="val")

        discloss, log_dict_disc = self.loss(qloss, x, xrec, 1, self.global_step,
                                            last_layer=self.get_last_layer(), split="val")
        rec_loss = log_dict_ae["val/rec_loss"]
        self.log("val/rec_loss", rec_loss,
                 prog_bar=True, logger=True, on_step=False, on_epoch=True, sync_dist=True)
        self.log("val/aeloss", aeloss,
                 prog_bar=True, logger=True, on_step=False, on_epoch=True, sync_dist=True)
        self.log_dict(log_dict_ae)
        self.log_dict(log_dict_disc)
        return self.log_dict

    def log_images(self, batch, **kwargs):
        log = dict()
        x = self.get_input(batch, self.image_key)
        x = x.to(self.device)
        # encode
        h = self.encoder(x)
        h = self.quant_conv(h)
        quant, _, _ = self.quantize(h)
        # decode
        x_rec = self.decode(quant)
        log["inputs"] = x
        log["reconstructions"] = x_rec
        return log


class EMAVQ(VQModel):
    def __init__(self,
                 ddconfig,
                 lossconfig,
                 n_embed,
                 embed_dim,
                 ckpt_path=None,
                 ignore_keys=[],
                 image_key="image",
                 colorize_nlabels=None,
                 monitor=None,
                 remap=None,
                 sane_index_shape=False,  # tell vector quantizer to return indices as bhw
                 ):
        super().__init__(ddconfig,
                         lossconfig,
                         n_embed,
                         embed_dim,
                         ckpt_path=None,
                         ignore_keys=ignore_keys,
                         image_key=image_key,
                         colorize_nlabels=colorize_nlabels,
                         monitor=monitor,
                         )
        self.quantize = EMAVectorQuantizer(n_embed=n_embed,
                                           embedding_dim=embed_dim,
                                           beta=0.25,
                                           remap=remap)

    def configure_optimizers(self):
        lr = self.learning_rate
        # Remove self.quantize from parameter list since it is updated via EMA
        opt_ae = torch.optim.Adam(list(self.encoder.parameters())+
                                  list(self.decoder.parameters())+
                                  list(self.quant_conv.parameters())+
                                  list(self.post_quant_conv.parameters()),
                                  lr=lr, betas=(0.5, 0.9))
        opt_disc = torch.optim.Adam(self.loss.discriminator.parameters(),
                                    lr=lr, betas=(0.5, 0.9))
        return [opt_ae, opt_disc], []
# tell vector quantizer to return indices as bhw # autoencode # discriminator # colorize with random projection # colorize with random projection # convert logits to indices # autoencode # annealing of temp # autoencode # discriminator # encode # decode # tell vector quantizer to return indices as bhw # Remove self.quantize from parameter list since it is updated via EMA