<reponame>UTokyo-ICEPP/multiml_htautau
import torch
import gpytorch
from gpytorch.distributions import MultitaskMultivariateNormal


class MultitaskGPModel(gpytorch.models.ApproximateGP):
    def __init__(self, inducing_points_num, input_dim, num_latents, num_tasks=8):
        # Let's use a different set of inducing points for each latent function
        # num_latents x inducing_points(in each dim) x input_dim
        inducing_points = torch.rand(num_latents, inducing_points_num, input_dim)

        # We have to mark the CholeskyVariationalDistribution as batch
        # so that we learn a variational distribution for each task
        variational_distribution = gpytorch.variational.CholeskyVariationalDistribution(
            inducing_points.size(-2), batch_shape=torch.Size([num_latents])
        )

        # We have to wrap the VariationalStrategy in a LMCVariationalStrategy
        # so that the output will be a MultitaskMultivariateNormal rather than a batch output
        variational_strategy = gpytorch.variational.LMCVariationalStrategy(
            gpytorch.variational.VariationalStrategy(
                self, inducing_points, variational_distribution, learn_inducing_locations=True
            ),
            num_tasks=num_tasks,
            num_latents=num_latents,
            latent_dim=-1
        )

        super().__init__(variational_strategy)

        # The mean and covariance modules should be marked as batch
        # so we learn a different set of hyperparameters
        self.mean_module = gpytorch.means.ConstantMean(batch_shape=torch.Size([num_latents]))
        # self.covar_module = gpytorch.kernels.RBFKernel(batch_shape=torch.Size([num_latents]))
        self.covar_module = gpytorch.kernels.ScaleKernel(
            gpytorch.kernels.RBFKernel(batch_shape=torch.Size([num_latents])),
            batch_shape=torch.Size([num_latents])
        )

    def forward(self, x):
        # The forward function should be written as if we were dealing with each output
        # dimension in batch
        mean_x = self.mean_module(x)
        covar_x = self.covar_module(x)
        return gpytorch.distributions.MultivariateNormal(mean_x, covar_x)


class IndependentMultitaskGPModel(gpytorch.models.ApproximateGP):
    def __init__(self, inducing_points_num, input_dim, num_tasks=8):
        # Let's use a different set of inducing points for each task
        inducing_points = torch.rand(num_tasks, inducing_points_num, input_dim)

        # We have to mark the CholeskyVariationalDistribution as batch
        # so that we learn a variational distribution for each task
        variational_distribution = gpytorch.variational.CholeskyVariationalDistribution(
            inducing_points.size(-2), batch_shape=torch.Size([num_tasks])
        )

        variational_strategy = gpytorch.variational.IndependentMultitaskVariationalStrategy(
            gpytorch.variational.VariationalStrategy(
                self, inducing_points, variational_distribution, learn_inducing_locations=True
            ),
            num_tasks=num_tasks,
        )

        super().__init__(variational_strategy)

        # The mean and covariance modules should be marked as batch
        # so we learn a different set of hyperparameters
        self.mean_module = gpytorch.means.ConstantMean(batch_shape=torch.Size([num_tasks]))
        self.covar_module = gpytorch.kernels.ScaleKernel(
            gpytorch.kernels.RBFKernel(batch_shape=torch.Size([num_tasks])),
            batch_shape=torch.Size([num_tasks])
        )

    def forward(self, x):
        # The forward function should be written as if we were dealing with each output
        # dimension in batch
        mean_x = self.mean_module(x)
        covar_x = self.covar_module(x)
        return gpytorch.distributions.MultivariateNormal(mean_x, covar_x)


def tensor_to_array(input_data):
    """ torch.tensor -> np.array
    Parameters
    ----------
    input_data : torch.Tensor
        torch.Tensor data
    Returns
    -------
    output_data : np.ndarray
        input_data converted to array type
    """
    output_data = input_data.to('cpu').detach().numpy().copy()
    return output_data

import torch
import gpytorch
from gpytorch.distributions import MultitaskMultivariateNormal


class MultitaskGPModel(gpytorch.models.ApproximateGP):
    def __init__(self, inducing_points_num, input_dim, num_latents, num_tasks=8):
        # Let's use a different set of inducing points for each latent function
        # num_latents x inducing_points(in each dim) x input_dim
        inducing_points = torch.rand(num_latents, inducing_points_num, input_dim)

        # We have to mark the CholeskyVariationalDistribution as batch
        # so that we learn a variational distribution for each task
        variational_distribution = gpytorch.variational.CholeskyVariationalDistribution(
            inducing_points.size(-2), batch_shape=torch.Size([num_latents])
        )

        # We have to wrap the VariationalStrategy in a LMCVariationalStrategy
        # so that the output will be a MultitaskMultivariateNormal rather than a batch output
        variational_strategy = gpytorch.variational.LMCVariationalStrategy(
            gpytorch.variational.VariationalStrategy(
                self, inducing_points, variational_distribution, learn_inducing_locations=True
            ),
            num_tasks=num_tasks,
            num_latents=num_latents,
            latent_dim=-1
        )

        super().__init__(variational_strategy)

        # The mean and covariance modules should be marked as batch
        # so we learn a different set of hyperparameters
        self.mean_module = gpytorch.means.ConstantMean(batch_shape=torch.Size([num_latents]))
        # self.covar_module = gpytorch.kernels.RBFKernel(batch_shape=torch.Size([num_latents]))
        self.covar_module = gpytorch.kernels.ScaleKernel(
            gpytorch.kernels.RBFKernel(batch_shape=torch.Size([num_latents])),
            batch_shape=torch.Size([num_latents])
        )

    def forward(self, x):
        # The forward function should be written as if we were dealing with each output
        # dimension in batch
        mean_x = self.mean_module(x)
        covar_x = self.covar_module(x)
        return gpytorch.distributions.MultivariateNormal(mean_x, covar_x)


class IndependentMultitaskGPModel(gpytorch.models.ApproximateGP):
    def __init__(self, inducing_points_num, input_dim, num_tasks=8):
        # Let's use a different set of inducing points for each task
        inducing_points = torch.rand(num_tasks, inducing_points_num, input_dim)

        # We have to mark the CholeskyVariationalDistribution as batch
        # so that we learn a variational distribution for each task
        variational_distribution = gpytorch.variational.CholeskyVariationalDistribution(
            inducing_points.size(-2), batch_shape=torch.Size([num_tasks])
        )

        variational_strategy = gpytorch.variational.IndependentMultitaskVariationalStrategy(
            gpytorch.variational.VariationalStrategy(
                self, inducing_points, variational_distribution, learn_inducing_locations=True
            ),
            num_tasks=num_tasks,
        )

        super().__init__(variational_strategy)

        # The mean and covariance modules should be marked as batch
        # so we learn a different set of hyperparameters
        self.mean_module = gpytorch.means.ConstantMean(batch_shape=torch.Size([num_tasks]))
        self.covar_module = gpytorch.kernels.ScaleKernel(
            gpytorch.kernels.RBFKernel(batch_shape=torch.Size([num_tasks])),
            batch_shape=torch.Size([num_tasks])
        )

    def forward(self, x):
        # The forward function should be written as if we were dealing with each output
        # dimension in batch
        mean_x = self.mean_module(x)
        covar_x = self.covar_module(x)
        return gpytorch.distributions.MultivariateNormal(mean_x, covar_x)


def tensor_to_array(input_data):
    """ torch.tensor -> np.array
    Parameters
    ----------
    input_data : torch.Tensor
        torch.Tensor data
    Returns
    -------
    output_data : np.ndarray
        input_data converted to array type
    """
    output_data = input_data.to('cpu').detach().numpy().copy()
    return output_data
# Let's use a different set of inducing points for each latent function # num_latents x inducing_points(in each dim) x input_dim # We have to mark the CholeskyVariationalDistribution as batch # so that we learn a variational distribution for each task # We have to wrap the VariationalStrategy in a LMCVariationalStrategy # so that the output will be a MultitaskMultivariateNormal rather than a batch output # The mean and covariance modules should be marked as batch # so we learn a different set of hyperparameters # self.covar_module = gpytorch.kernels.RBFKernel(batch_shape=torch.Size([num_latents])) # The forward function should be written as if we were dealing with each output # dimension in batch # Let's use a different set of inducing points for each task # We have to mark the CholeskyVariationalDistribution as batch # so that we learn a variational distribution for each task # The mean and covariance modules should be marked as batch # so we learn a different set of hyperparameters # The forward function should be written as if we were dealing with each output # dimension in batch torch.tensor -> np.array     Parameters     ----------     input_data : torch.Tensor         torch.Tensor data     Returns     -------     output_data : np.ndarray         input_data converted to array type