title: How to get rid of checkerboard artifacts
body: <p><strong>Hello fellow coders,</strong></p>
  
  <p>I am using a fully convolutional autoencoder to color black and white images, however, the output has a <a href="https://distill.pub/2016/deconv-checkerboard/" rel="nofollow noreferrer">checkerboard pattern</a> and I want to get rid of it. The checkerboard artifacts I have seen so far allways have been far smaller than mine and the usual way to get rid of them is replacing all unpooling operations with bilinear upsampling (I have been told that). </p>
  
  <p>But I can not simply replace the unpooling operation because I work with different sized images, thus the unpooling operation is needed, else the output tensor could have a different size than the original. </p>
  
  <p><strong>TLDR:</strong></p>
  
  <p>How can I get rid of these checkerboard-artifacts without replacing the unpooling operations?</p>
  
  <pre><code>class AE(nn.Module):
      def __init__(self):
          super(AE, self).__init__()
          self.leaky_reLU = nn.LeakyReLU(0.2)
          self.pool = nn.MaxPool2d(kernel_size=2, stride=2, padding=1, return_indices=True)
          self.unpool = nn.MaxUnpool2d(kernel_size=2, stride=2, padding=1)
          self.softmax = nn.Softmax2d()
  
          self.conv1 = nn.Conv2d(in_channels=3, out_channels=64, kernel_size=3, stride=1, padding=1)
          self.conv2 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, stride=1, padding=1)
          self.conv3 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=1, padding=1)
          self.conv4 = nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3, stride=1, padding=1)
          self.conv5 = nn.Conv2d(in_channels=512, out_channels=1024, kernel_size=3, stride=1, padding=1)
          self.conv6 = nn.ConvTranspose2d(in_channels=1024, out_channels=512, kernel_size=3, stride=1, padding=1)
          self.conv7 = nn.ConvTranspose2d(in_channels=512, out_channels=256, kernel_size=3, stride=1, padding=1)
          self.conv8 = nn.ConvTranspose2d(in_channels=256, out_channels=128, kernel_size=3, stride=1, padding=1)
          self.conv9 = nn.ConvTranspose2d(in_channels=128, out_channels=64, kernel_size=3, stride=1, padding=1)
          self.conv10 = nn.ConvTranspose2d(in_channels=64, out_channels=2, kernel_size=3, stride=1, padding=1)
  
      def forward(self, x):
  
          # encoder
          x = self.conv1(x)
          x = self.leaky_reLU(x)
          size1 = x.size()
          x, indices1 = self.pool(x)
  
          x = self.conv2(x)
          x = self.leaky_reLU(x)
          size2 = x.size()
          x, indices2 = self.pool(x)
  
          x = self.conv3(x)
          x = self.leaky_reLU(x)
          size3 = x.size()
          x, indices3 = self.pool(x)
  
          x = self.conv4(x)
          x = self.leaky_reLU(x)
          size4 = x.size()
          x, indices4 = self.pool(x)
  
          ######################
          x = self.conv5(x)
          x = self.leaky_reLU(x)
  
          x = self.conv6(x)
          x = self.leaky_reLU(x)
          ######################
  
          # decoder
          x = self.unpool(x, indices4, output_size=size4)
          x = self.conv7(x)
          x = self.leaky_reLU(x)
  
          x = self.unpool(x, indices3, output_size=size3)
          x = self.conv8(x)
          x = self.leaky_reLU(x)
  
          x = self.unpool(x, indices2, output_size=size2)
          x = self.conv9(x)
          x = self.leaky_reLU(x)
  
          x = self.unpool(x, indices1, output_size=size1)
          x = self.conv10(x)
          x = self.softmax(x)
  
          return x
  </code></pre>
  
  <p><a href="https://i.stack.imgur.com/Y5Lv8.png" rel="nofollow noreferrer"><img src="https://i.stack.imgur.com/Y5Lv8.png" alt="NNs output"></a></p>
  
  <p>.</p>
  
  <h2>EDIT - Solution:</h2>
  
  <p>Skip-Connections are the way to go!</p>
  
  <pre><code>class AE(nn.Module):
      def __init__(self):
          super(AE, self).__init__()
          self.leaky_reLU = nn.LeakyReLU(0.2)
          self.pool = nn.MaxPool2d(kernel_size=2, stride=2, padding=1, return_indices=True)
          self.unpool = nn.MaxUnpool2d(kernel_size=2, stride=2, padding=1)
          self.softmax = nn.Softmax2d()
  
          self.conv1 = nn.Conv2d(in_channels=3, out_channels=64, kernel_size=3, stride=1, padding=1)
          self.conv2 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, stride=1, padding=1)
          self.conv3 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=1, padding=1)
          self.conv4 = nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3, stride=1, padding=1)
          self.conv5 = nn.Conv2d(in_channels=512, out_channels=1024, kernel_size=3, stride=1, padding=1)
          self.conv6 = nn.Conv2d(in_channels=1024, out_channels=512, kernel_size=3, stride=1, padding=1)
          self.conv7 = nn.Conv2d(in_channels=1024, out_channels=256, kernel_size=3, stride=1, padding=1)
          self.conv8 = nn.Conv2d(in_channels=512, out_channels=128, kernel_size=3, stride=1, padding=1)
          self.conv9 = nn.Conv2d(in_channels=256, out_channels=64, kernel_size=3, stride=1, padding=1)
          self.conv10 = nn.Conv2d(in_channels=128, out_channels=2, kernel_size=3, stride=1, padding=1)
  
      def forward(self, x):
  
          # encoder
          x = self.conv1(x)
          out1 = self.leaky_reLU(x)
          x = out1
          size1 = x.size()
          x, indices1 = self.pool(x)
  
          x = self.conv2(x)
          out2 = self.leaky_reLU(x)
          x = out2
          size2 = x.size()
          x, indices2 = self.pool(x)
  
          x = self.conv3(x)
          out3 = self.leaky_reLU(x)
          x = out3
          size3 = x.size()
          x, indices3 = self.pool(x)
  
          x = self.conv4(x)
          out4 = self.leaky_reLU(x)
          x = out4
          size4 = x.size()
          x, indices4 = self.pool(x)
  
          ######################
          x = self.conv5(x)
          x = self.leaky_reLU(x)
  
          x = self.conv6(x)
          x = self.leaky_reLU(x)
          ######################
  
          # decoder
          x = self.unpool(x, indices4, output_size=size4)
          x = self.conv7(torch.cat((x, out4), 1))
          x = self.leaky_reLU(x)
  
          x = self.unpool(x, indices3, output_size=size3)
          x = self.conv8(torch.cat((x, out3), 1))
          x = self.leaky_reLU(x)
  
          x = self.unpool(x, indices2, output_size=size2)
          x = self.conv9(torch.cat((x, out2), 1))
          x = self.leaky_reLU(x)
  
          x = self.unpool(x, indices1, output_size=size1)
          x = self.conv10(torch.cat((x, out1), 1))
          x = self.softmax(x)
  
          return x
  </code></pre>