manga_translator/inpainting/inpainting_attn.py

from typing import List, Optional
import torch
import torch.nn as nn
import torch.nn.functional as F

import numpy as np

def relu_nf(x):
    return F.relu(x) * 1.7139588594436646

def gelu_nf(x):
    return F.gelu(x) * 1.7015043497085571

def silu_nf(x):
    return F.silu(x) * 1.7881293296813965

class LambdaLayer(nn.Module):
    def __init__(self, f):
        super(LambdaLayer, self).__init__()
        self.f = f

    def forward(self, x):
        return self.f(x)

class ScaledWSConv2d(nn.Conv2d):
    """2D Conv layer with Scaled Weight Standardization."""
    def __init__(self, in_channels, out_channels, kernel_size,
        stride=1, padding=0,
        dilation=1, groups=1, bias=True, gain=True,
        eps=1e-4):
        nn.Conv2d.__init__(self, in_channels, out_channels,
            kernel_size, stride,
            padding, dilation,
            groups, bias)
        #nn.init.kaiming_normal_(self.weight)
        if gain:
            self.gain = nn.Parameter(torch.ones(self.out_channels, 1, 1, 1))
        else:
            self.gain = None
        # Epsilon, a small constant to avoid dividing by zero.
        self.eps = eps
    def get_weight(self):
        # Get Scaled WS weight OIHW;
        fan_in = np.prod(self.weight.shape[1:])
        var, mean = torch.var_mean(self.weight, dim=(1, 2, 3), keepdims=True)
        scale = torch.rsqrt(torch.max(
            var * fan_in, torch.tensor(self.eps).to(var.device))) * self.gain.view_as(var).to(var.device)
        shift = mean * scale
        return self.weight * scale - shift

    def forward(self, x):
        return F.conv2d(x, self.get_weight(), self.bias,
            self.stride, self.padding,
            self.dilation, self.groups)

class ScaledWSTransposeConv2d(nn.ConvTranspose2d):
    """2D Transpose Conv layer with Scaled Weight Standardization."""
    def __init__(self, in_channels: int,
        out_channels: int,
        kernel_size,
        stride = 1,
        padding = 0,
        output_padding = 0,
        groups: int = 1,
        bias: bool = True,
        dilation: int = 1,
        gain=True,
        eps=1e-4):
        nn.ConvTranspose2d.__init__(self, in_channels, out_channels, kernel_size, stride, padding, output_padding, groups, bias, dilation, 'zeros')
        #nn.init.kaiming_normal_(self.weight)
        if gain:
            self.gain = nn.Parameter(torch.ones(self.in_channels, 1, 1, 1))
        else:
            self.gain = None
        # Epsilon, a small constant to avoid dividing by zero.
        self.eps = eps
    def get_weight(self):
        # Get Scaled WS weight OIHW;
        fan_in = np.prod(self.weight.shape[1:])
        var, mean = torch.var_mean(self.weight, dim=(1, 2, 3), keepdims=True)
        scale = torch.rsqrt(torch.max(
            var * fan_in, torch.tensor(self.eps).to(var.device))) * self.gain.view_as(var).to(var.device)
        shift = mean * scale
        return self.weight * scale - shift

    def forward(self, x, output_size: Optional[List[int]] = None):
        output_padding = self._output_padding(
            input, output_size, self.stride, self.padding, self.kernel_size, self.dilation)
        return F.conv_transpose2d(x, self.get_weight(), self.bias, self.stride, self.padding,
            output_padding, self.groups, self.dilation)

class GatedWSConvPadded(nn.Module):
    def __init__(self, in_ch, out_ch, ks, stride = 1, dilation = 1):
        super(GatedWSConvPadded, self).__init__()
        self.in_ch = in_ch
        self.out_ch = out_ch
        self.padding = nn.ReflectionPad2d((ks - 1) // 2)
        self.conv = ScaledWSConv2d(in_ch, out_ch, kernel_size = ks, stride = stride)
        self.conv_gate = ScaledWSConv2d(in_ch, out_ch, kernel_size = ks, stride = stride)

    def forward(self, x):
        x = self.padding(x)
        signal = self.conv(x)
        gate = torch.sigmoid(self.conv_gate(x))
        return signal * gate * 1.8

class GatedWSTransposeConvPadded(nn.Module):
    def __init__(self, in_ch, out_ch, ks, stride = 1):
        super(GatedWSTransposeConvPadded, self).__init__()
        self.in_ch = in_ch
        self.out_ch = out_ch
        self.conv = ScaledWSTransposeConv2d(in_ch, out_ch, kernel_size = ks, stride = stride, padding = (ks - 1) // 2)
        self.conv_gate = ScaledWSTransposeConv2d(in_ch, out_ch, kernel_size = ks, stride = stride, padding = (ks - 1) // 2)

    def forward(self, x):
        signal = self.conv(x)
        gate = torch.sigmoid(self.conv_gate(x))
        return signal * gate * 1.8

class ResBlock(nn.Module):
    def __init__(self, ch, alpha = 0.2, beta = 1.0, dilation = 1):
        super(ResBlock, self).__init__()
        self.alpha = alpha
        self.beta = beta
        self.c1 = GatedWSConvPadded(ch, ch, 3, dilation = dilation)
        self.c2 = GatedWSConvPadded(ch, ch, 3, dilation = dilation)

    def forward(self, x):
        skip = x
        x = self.c1(relu_nf(x / self.beta))
        x = self.c2(relu_nf(x))
        x = x * self.alpha
        return x + skip

# from https://github.com/SayedNadim/Global-and-Local-Attention-Based-Free-Form-Image-Inpainting
class GlobalAttention(nn.Module):
    """ Self attention Layer"""

    def __init__(self, in_dim):
        super(GlobalAttention, self).__init__()
        self.channel_in = in_dim

        self.query_conv = ScaledWSConv2d(in_channels=in_dim, out_channels=in_dim, kernel_size=1)
        self.key_conv = ScaledWSConv2d(in_channels=in_dim, out_channels=in_dim, kernel_size=1)
        self.value_conv = ScaledWSConv2d(in_channels=in_dim, out_channels=in_dim, kernel_size=1)
        self.softmax = nn.Softmax(dim=-1)  #
        self.rate = 1
        self.gamma = nn.parameter.Parameter(torch.tensor([1.0], requires_grad=True), requires_grad=True)

    def forward(self, a, b, c):
        m_batchsize, C, height, width = a.size()  # B, C, H, W
        c = F.interpolate(c, size=(height, width), mode='nearest')
        proj_query = self.query_conv(a).view(m_batchsize, -1, width * height).permute(0, 2, 1)  # B, C, N -> B N C
        proj_key = self.key_conv(b).view(m_batchsize, -1, width * height)  # B, C, N
        feature_similarity = torch.bmm(proj_query, proj_key)  # B, N, N

        mask = c.view(m_batchsize, -1, width * height)  # B, C, N
        mask = mask.repeat(1, height * width, 1).permute(0, 2, 1)  # B, 1, H, W -> B, C, H, W // B

        feature_pruning = feature_similarity * mask
        attention = self.softmax(feature_pruning)  # B, N, C
        feature_pruning = torch.bmm(self.value_conv(a).view(m_batchsize, -1, width * height),
                                    attention.permute(0, 2, 1))  # -. B, C, N
        out = feature_pruning.view(m_batchsize, C, height, width)  # B, C, H, W
        out = a * c + self.gamma *  (1.0 - c) * out
        return out

class CoarseGenerator(nn.Module):
    def __init__(self, in_ch = 4, out_ch = 3, ch = 32, alpha = 0.2):
        super(CoarseGenerator, self).__init__()

        self.head = nn.Sequential(
            GatedWSConvPadded(in_ch, ch, 3, stride = 1),
            LambdaLayer(relu_nf),
            GatedWSConvPadded(ch, ch * 2, 4, stride = 2),
            LambdaLayer(relu_nf),
            GatedWSConvPadded(ch * 2, ch * 4, 4, stride = 2),
        )

        self.beta = 1.0
        self.alpha = alpha
        self.body_conv = []
        self.body_conv.append(ResBlock(ch * 4, self.alpha, self.beta))
        self.beta = (self.beta ** 2 + self.alpha ** 2) ** 0.5
        self.body_conv.append(ResBlock(ch * 4, self.alpha, self.beta))
        self.beta = (self.beta ** 2 + self.alpha ** 2) ** 0.5
        self.body_conv.append(ResBlock(ch * 4, self.alpha, self.beta, 2))
        self.beta = (self.beta ** 2 + self.alpha ** 2) ** 0.5
        self.body_conv.append(ResBlock(ch * 4, self.alpha, self.beta, 4))
        self.beta = (self.beta ** 2 + self.alpha ** 2) ** 0.5
        self.body_conv.append(ResBlock(ch * 4, self.alpha, self.beta, 8))
        self.beta = (self.beta ** 2 + self.alpha ** 2) ** 0.5
        self.body_conv.append(ResBlock(ch * 4, self.alpha, self.beta, 16))
        self.beta = (self.beta ** 2 + self.alpha ** 2) ** 0.5
        self.body_conv = nn.Sequential(*self.body_conv)

        self.tail = nn.Sequential(
            LambdaLayer(relu_nf),
            GatedWSConvPadded(ch * 8, ch * 8, 3, 1),
            LambdaLayer(relu_nf),
            GatedWSConvPadded(ch * 8, ch * 4, 3, 1),
            LambdaLayer(relu_nf),
            GatedWSConvPadded(ch * 4, ch * 4, 3, 1),
            LambdaLayer(relu_nf),
            GatedWSConvPadded(ch * 4, ch * 4, 3, 1),
            LambdaLayer(relu_nf),
            GatedWSTransposeConvPadded(ch * 4, ch * 2, 4, 2),
            LambdaLayer(relu_nf),
            GatedWSTransposeConvPadded(ch * 2, ch, 4, 2),
            LambdaLayer(relu_nf),
            GatedWSConvPadded(ch, out_ch, 3, stride = 1),
        )

        self.beta = 1.0

        self.body_attn_1 = ResBlock(ch * 4, self.alpha, self.beta)
        self.beta = (self.beta ** 2 + self.alpha ** 2) ** 0.5
        self.body_attn_2 = ResBlock(ch * 4, self.alpha, self.beta)
        self.beta = (self.beta ** 2 + self.alpha ** 2) ** 0.5
        self.body_attn_3 = ResBlock(ch * 4, self.alpha, self.beta)
        self.beta = (self.beta ** 2 + self.alpha ** 2) ** 0.5
        self.body_attn_4 = ResBlock(ch * 4, self.alpha, self.beta)
        self.beta = (self.beta ** 2 + self.alpha ** 2) ** 0.5
        self.body_attn_attn = GlobalAttention(in_dim = ch * 4)
        self.body_attn_5 = ResBlock(ch * 4, self.alpha, self.beta)
        self.beta = (self.beta ** 2 + self.alpha ** 2) ** 0.5
        self.body_attn_6 = ResBlock(ch * 4, self.alpha, self.beta)
        self.beta = (self.beta ** 2 + self.alpha ** 2) ** 0.5

    def forward(self, img, mask):
        x = torch.cat([mask, img], dim = 1)
        x = self.head(x)
        attn = self.body_attn_1(x)
        attn = self.body_attn_2(attn)
        attn = self.body_attn_3(attn)
        attn = self.body_attn_4(attn)
        attn = self.body_attn_attn(attn, attn, mask)
        attn = self.body_attn_5(attn)
        attn = self.body_attn_6(attn)
        conv = self.body_conv(x)
        x = self.tail(torch.cat([conv, attn], dim = 1))
        return torch.clip(x, -1, 1)

class InpaintingVanilla(nn.Module):
    def __init__(self):
        super(InpaintingVanilla, self).__init__()
        self.coarse_generator = CoarseGenerator(4, 3, 32)

    def forward(self, x, mask):
        x_stage1 = self.coarse_generator(x, mask)
        return x_stage1