Uploaded app code

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+images/cat.jpeg filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,247 @@
+import gradio as gr
+import torch
+from torchvision import transforms
+import numpy as np
+from PIL import Image
+from pytorch_grad_cam import GradCAM
+from pytorch_grad_cam.utils.model_targets import ClassifierOutputTarget
+from pytorch_grad_cam.utils.image import show_cam_on_image
+import io
+from models import custom_resnet_lightning_s10
+from utils import load_model_from_checkpoint, denormalize, get_data_label_name, get_dataset_labels
+
+device = torch.device('cpu')
+dataset_mean, dataset_std = (0.4914, 0.4822, 0.4465), \
+                            (0.2470, 0.2435, 0.2616)
+model = custom_resnet_lightning_s10.S10LightningModel(64)
+
+checkpoint = load_model_from_checkpoint(device)
+model.load_state_dict(checkpoint['model'], strict=False)
+
+test_incorrect_pred = checkpoint['test_incorrect_pred']
+
+sample_images = [
+    ['images/aeroplane.jpeg', 0],
+    ['images/bird.jpeg', 2],
+    ['images/car.jpeg', 1],
+    ['images/cat.jpeg', 3],
+    ['images/deer.jpeg', 4],
+    ['images/dog.jpeg', 5],
+    ['images/frog.jpeg', 6],
+    ['images/horse.jpeg', 7],
+    ['images/ship.jpeg', 8],
+    ['images/truck.jpeg', 9]
+]
+
+with gr.Blocks() as app:
+    '''
+    Select feature interface
+    '''
+    with gr.Row() as input_radio_group:
+        radio_btn = gr.Radio(
+            choices=['Top Prediction Classes', 'GradCAM Images', 'Missclassified Images'],
+            type="index",
+            label='Feature options',
+            info="Choose which feature you want to explore",
+            value='Top Prediction Classes'
+        )
+
+    '''
+    Options for GradCAM feature
+    '''
+    with gr.Row():
+        with gr.Column(visible=False) as grad_cam_col:
+            grad_cam_count = gr.Slider(1, 20, value=5, step=1, label="Choose image count",
+                                       info="How many images you want to view?")
+            grad_cam_layer = gr.Slider(-4, -1, value=-3, step=1, label="Choose model layer",
+                                       info="Which layer you want to view GradCAM on? [-4 => last layer]")
+            grad_cam_opacity = gr.Slider(0, 1, value=0.4, step=0.1, label="Choose opacity of the gradient")
+
+            with gr.Column():
+                grad_cam_btn = gr.Button("Yes, Go Ahead")
+
+        with gr.Column(visible=False) as grad_cam_output:
+            grad_cam_output_gallery = gr.Gallery(value=[], columns=3, label='Output')
+            # prediction_title = gr.Label(value='')
+
+    '''
+    Options for Missclassfied images feature
+    '''
+    with gr.Row(visible=False) as missclassified_col:
+        with gr.Row():
+            missclassified_img_count = gr.Slider(1, 20, value=5, step=1, label="Choose image count",
+                                                 info="How man missclassified images you want to view?")
+            missclassified_btn = gr.Button("Click to Continue")
+    with gr.Row(visible=False) as missclassified_img_output:
+        missclassified_img_output_gallery = gr.Gallery(value=[], columns=5, label='Output')
+
+    '''
+    Option for Top prediction classes
+    '''
+    with gr.Row(visible=True) as top_pred_cls_col:
+        with gr.Column():
+            example_images = gr.Gallery(allow_preview=False, label='Select image ', info='',
+                                        value=[img[0] for img in sample_images], columns=3, rows=2,
+                                        object_fit='scale_down')
+
+        with gr.Column():
+            with gr.Row():
+                top_pred_image = gr.Image(shape=(32, 32), label='Upload Image or Select from the gallery')
+                top_class_count = gr.Slider(1, 10, value=5, step=1, label="Number of classes to predict")
+                top_class_btn = gr.Button("Submit")
+
+        with gr.Row(visible=True) as top_class_output:
+            # top_class_output_img = gr.Image().style(width=256, height=256)
+            top_class_output_labels = gr.Label(num_top_classes=top_class_count.value, label='Output')
+
+
+    def on_select(evt: gr.SelectData):
+        return {
+            top_pred_image: sample_images[evt.index][0]
+        }
+
+
+    example_images.select(on_select, None, top_pred_image)
+
+
+    def top_class_img_upload(input_img, top_class_count):
+        if input_img is not None:
+            transform = transforms.ToTensor()
+            org_img = input_img
+            input_img = transform(input_img)
+            input_img = input_img.to(device)
+            input_img = input_img.unsqueeze(0)
+            outputs = model(input_img, no_softmax=True)
+            softmax = torch.nn.Softmax(dim=0)
+            o = softmax(outputs.flatten())
+            confidences = {get_dataset_labels()[i]: float(o[i]) for i in range(10)}
+            top_class_output_labels.num_top_classes = top_class_count
+            return {
+                top_class_output: gr.update(visible=True),
+                # top_class_output_img: org_img,
+                top_class_output_labels: confidences
+            }
+
+
+    top_class_btn.click(
+        top_class_img_upload,
+        [top_pred_image, top_class_count],
+        [top_class_output, top_class_output_labels]
+    )
+
+    '''
+    Missclassified Images feature
+    '''
+
+
+    def show_missclassified_images(img_count):
+        imgs = []
+        for i in range(img_count):
+            img = test_incorrect_pred['images'][i].cpu()
+            img = denormalize(img, dataset_mean, dataset_std)
+            img = np.array(255 * img, np.int16).transpose(1, 2, 0)
+            label = '✅ ' + get_data_label_name(
+                test_incorrect_pred['ground_truths'][i].item()) + ' ❌ ' + get_data_label_name(
+                test_incorrect_pred['predicted_vals'][i].item())
+            imgs.append((img, label))
+
+        return {
+            missclassified_img_output: gr.update(visible=True),
+            missclassified_img_output_gallery: imgs
+        }
+
+
+    missclassified_btn.click(
+        show_missclassified_images,
+        [missclassified_img_count],
+        [missclassified_img_output_gallery, missclassified_img_output]
+    )
+
+    '''
+    GradCAM Feature
+    '''
+
+
+    def grad_cam_submit(img_count, layer_idx, grad_opacity):
+
+        target_layers = [model.get_layer(-1 * (layer_idx + 1))]
+        cam = GradCAM(model=model, target_layers=target_layers)
+
+        visual_arr = []
+        pred_arr = []
+        for i in range(img_count):
+            pred_dict = test_incorrect_pred
+
+            targets = [ClassifierOutputTarget(pred_dict['ground_truths'][i].cpu().item())]
+
+            grayscale_cam = cam(input_tensor=pred_dict['images'][i][None, :].cpu(), targets=targets)
+
+            x = denormalize(pred_dict['images'][i].cpu(), dataset_mean, dataset_std)
+
+            image = np.array(255 * x, np.int16).transpose(1, 2, 0)
+            img_tensor = np.array(x, np.float16).transpose(1, 2, 0)
+
+            visualization = show_cam_on_image(img_tensor, grayscale_cam.transpose(1, 2, 0), use_rgb=True,
+                                              image_weight=(1.0 - grad_opacity))
+
+            visual_arr.append(
+                (visualization, get_data_label_name(pred_dict['ground_truths'][i].item()))
+            )
+
+        return {
+            grad_cam_output: gr.update(visible=True),
+            grad_cam_output_gallery: visual_arr
+        }
+
+
+    grad_cam_btn.click(
+        grad_cam_submit,
+        [grad_cam_count, grad_cam_layer, grad_cam_opacity],
+        [grad_cam_output_gallery, grad_cam_output]
+    )
+
+    '''
+    Select Feature to showcase
+    '''
+
+
+    def select_feature(feature):
+        if feature == 0:
+            return {
+                grad_cam_col: gr.update(visible=False),
+                grad_cam_output: gr.update(visible=False),
+                missclassified_col: gr.update(visible=False),
+                missclassified_img_output: gr.update(visible=False),
+                top_pred_cls_col: gr.update(visible=True),
+                top_class_output: gr.update(visible=True)
+            }
+        elif feature == 1:
+            return {
+                grad_cam_col: gr.update(visible=False),
+                grad_cam_output: gr.update(visible=False),
+                missclassified_col: gr.update(visible=True),
+                missclassified_img_output: gr.update(visible=True),
+                top_pred_cls_col: gr.update(visible=False),
+                top_class_output: gr.update(visible=False)
+            }
+
+        else:
+            return {
+                grad_cam_col: gr.update(visible=True),
+                grad_cam_output: gr.update(visible=True),
+                missclassified_col: gr.update(visible=False),
+                missclassified_img_output: gr.update(visible=False),
+                top_pred_cls_col: gr.update(visible=False),
+                top_class_output: gr.update(visible=False)
+            }
+
+
+    radio_btn.change(select_feature,
+                     [radio_btn],
+                     [grad_cam_col, grad_cam_output, missclassified_col, missclassified_img_output, top_pred_cls_col,
+                      top_class_output])
+
+'''
+Launch the app
+'''
+app.launch()

ckpt.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c5cef3f797917b1f454d5538e8d39af1dea5a0dd880a148e5c19a1b1c746263
+size 88712703

models/custom_resnet_lightning_s10.py

@@ -0,0 +1,324 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import pytorch_lightning as pl
+from pytorch_grad_cam import GradCAM
+from pytorch_grad_cam.utils.model_targets import ClassifierOutputTarget
+from pytorch_grad_cam.utils.image import show_cam_on_image
+
+import matplotlib.pyplot as plt
+from torch_lr_finder import LRFinder
+import numpy as np
+from utils import get_correct_pred_count, add_predictions, test_incorrect_pred, test_correct_pred, denormalize
+
+NO_GROUPS = 4
+class ResnetBlock(nn.Module):
+    def __init__(self, input_channel, output_channel, padding=1, norm='bn', drop=0.01):
+
+        super(ResnetBlock, self).__init__()
+
+        self.conv1 = nn.Conv2d(input_channel, output_channel, 3, padding=padding)
+
+        if norm == 'bn':
+            self.n1 = nn.BatchNorm2d(output_channel)
+        elif norm == 'gn':
+            self.n1 = nn.GroupNorm(NO_GROUPS, output_channel)
+        elif norm == 'ln':
+            self.n1 = nn.GroupNorm(1, output_channel)
+
+        self.drop1 = nn.Dropout2d(drop)
+
+        self.conv2 = nn.Conv2d(output_channel, output_channel, 3, padding=padding)
+
+        if norm == 'bn':
+            self.n2 = nn.BatchNorm2d(output_channel)
+        elif norm == 'gn':
+            self.n2 = nn.GroupNorm(NO_GROUPS, output_channel)
+        elif norm == 'ln':
+            self.n2 = nn.GroupNorm(1, output_channel)
+
+        self.drop2 = nn.Dropout2d(drop)
+
+
+    '''
+    Depending on the model requirement, Convolution block with number of layers is applied to the input image
+    '''
+    def __call__(self, x):
+
+        x = self.conv1(x)
+        x = self.n1(x)
+        x = F.relu(x)
+
+        x = self.drop1(x)
+
+
+        #if layers >= 2:
+
+        x = self.conv2(x)
+
+        x = self.n2(x)
+        x = F.relu(x)
+        x = self.drop2(x)
+
+        return x
+
+
+class S10LightningModel(pl.LightningModule):
+    def __init__(self, base_channels, drop=0.01, loss_function=F.cross_entropy, is_find_max_lr=False, max_lr=3.20E-04):
+        super(S10LightningModel, self).__init__()
+
+        self.is_find_max_lr = is_find_max_lr
+        self.max_lr = max_lr
+        self.criterion = loss_function
+
+        self.metric = dict(train=0,
+                        val=0,
+                        train_total=0,
+                        val_total=0,
+                        epoch_train_loss=[],
+                        epoch_val_loss=[],
+                        train_loss=[],
+                        val_loss=[],
+                        train_acc=[],
+                        val_acc=[])
+
+        self.base_channels = base_channels
+
+        self.prep_layer = nn.Sequential(
+            nn.Conv2d(3, base_channels, 3, stride=1, padding=1),
+            nn.BatchNorm2d(base_channels),
+            nn.ReLU(),
+            nn.Dropout2d(drop)
+        )
+
+        # layer1
+        self.x1 = nn.Sequential(
+            nn.Conv2d(base_channels, 2 * base_channels, 3, stride=1, padding=1),
+            nn.MaxPool2d(2, 2),
+            nn.BatchNorm2d(2 * base_channels),
+            nn.ReLU(),
+            nn.Dropout2d(drop)
+        )
+
+        self.R1 = ResnetBlock(2 * base_channels, 2 * base_channels, padding=1, drop=drop)
+
+        # layer2
+        self.layer2 = nn.Sequential(
+            nn.Conv2d(2 * base_channels, 4 * base_channels, 3, stride=1, padding=1),
+            nn.MaxPool2d(2, 2),
+            nn.BatchNorm2d(4 * base_channels),
+            nn.ReLU(),
+            nn.Dropout2d(drop)
+        )
+
+        # layer3
+        self.x2 = nn.Sequential(
+            nn.Conv2d(4 * base_channels, 8 * base_channels, 3, stride=1, padding=1),
+            nn.MaxPool2d(2, 2),
+            nn.BatchNorm2d(8 * base_channels),
+            nn.ReLU(),
+            nn.Dropout2d(drop)
+        )
+
+        self.R2 = ResnetBlock(8 * base_channels, 8 * base_channels, padding=1, drop=drop)
+
+        self.pool = nn.MaxPool2d(4)
+
+        self.fc = nn.Linear(8 * base_channels, 10)
+
+    def forward(self, x, no_softmax=False):
+
+        # print(x.size())
+
+        x = self.prep_layer(x)
+        # print(x.size())
+
+        x = self.x1(x)
+        # print('x1', x.size())
+
+        x = self.R1(x) + x
+        # print('x', x.size())
+
+        x = self.layer2(x)
+        # print(x.size())
+
+        x = self.x2(x)
+        # print('x2', x.size())
+
+        x = self.R2(x) + x
+        # print('x', x.size())
+
+        x = self.pool(x)
+        # print(x.size())
+
+        x = x.view(x.size(0), 8 * self.base_channels)
+        # print(x.size())
+
+        x = self.fc(x)
+        # print(x.size())
+
+        if no_softmax:
+            print(x.size())
+            return x
+
+        return F.log_softmax(x, dim=1)
+
+
+    def get_layer(self, idx):
+        layers = [self.prep_layer, self.x1, self.layer2, self.x2, self.pool]
+
+        if idx < len(layers) and idx >= 0:
+            return layers[idx]
+
+
+    def training_step(self, train_batch, batch_idx):
+        x, target = train_batch
+        output = self.forward(x)
+        loss = self.criterion(output, target)
+
+        self.metric['train'] += get_correct_pred_count(output, target)
+        self.metric['train_total'] += len(x)
+        self.metric['epoch_train_loss'].append(loss)
+
+        acc = 100 * self.metric['train'] / self.metric['train_total']
+
+        self.log_dict({'train_loss': loss, 'train_acc': acc})
+        return loss
+
+
+    def validation_step(self, val_batch, batch_idx):
+        x, target = val_batch
+        output = self.forward(x)
+        loss = self.criterion(output, target)
+
+        self.metric['val'] += get_correct_pred_count(output, target)
+        self.metric['val_total'] += len(x)
+        self.metric['epoch_val_loss'].append(loss)
+
+        acc = 100 * self.metric['val'] / self.metric['val_total']
+
+        if self.current_epoch == self.trainer.max_epochs - 1:
+            add_predictions(x, output, target)
+
+        self.log_dict({'val_loss': loss, 'val_acc': acc})
+
+
+    def test_step(self, test_batch, batch_idx):
+        self.validation_step(test_batch, batch_idx)
+
+    def train_dataloader(self):
+        if not self.trainer.train_dataloader:
+            self.trainer.fit_loop.setup_data()
+
+        return self.trainer.train_dataloader
+
+    def configure_optimizers(self):
+        optimizer = torch.optim.Adam(self.parameters(), lr=1e-6, weight_decay=0.01)
+        self.find_lr(optimizer)
+        print(self.max_lr)
+        scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer,
+                                                  max_lr=self.max_lr,
+                                                  epochs=self.trainer.max_epochs,
+                                                  steps_per_epoch=len(self.train_dataloader()),
+                                                  pct_start=5 / self.trainer.max_epochs,
+                                                  div_factor=100,
+                                                  final_div_factor=100,
+                                                  three_phase=False,
+                                                  verbose=False
+                                                  )
+        return {
+            "optimizer": optimizer,
+            "lr_scheduler": {
+                "scheduler": scheduler,
+                'interval': 'step', # or 'epoch'
+                'frequency': 1
+            },
+        }
+
+
+    def on_validation_epoch_end(self):
+        if self.metric['train_total']:
+            print('Epoch ', self.current_epoch)
+            train_acc = 100 * self.metric['train'] / self.metric['train_total']
+            epoch_loss = sum(self.metric['epoch_train_loss']) / len(self.metric['epoch_train_loss'])
+            self.metric['train_loss'].append( epoch_loss.item() )
+            self.metric['train_acc'].append(train_acc)
+
+
+            print('Train Loss: ', epoch_loss.item(), ' Accuracy: ', str(train_acc) + '%', ' [',
+                  self.metric['train'], '/', self.metric['train_total'], ']')
+
+            self.metric['train'] = 0
+            self.metric['train_total'] = 0
+            self.metric['epoch_train_loss'] = []
+
+            val_acc = 100 * self.metric['val'] / self.metric['val_total']
+
+            epoch_loss = sum(self.metric['epoch_val_loss']) / len(self.metric['epoch_val_loss'])
+            self.metric['val_loss'].append( epoch_loss.item() )
+            self.metric['val_acc'].append(val_acc)
+
+            print('Validation Loss: ', epoch_loss.item(), ' Accuracy: ', str(val_acc) + '%', ' [', self.metric['val'],
+                  '/', self.metric['val_total'], ']\n')
+
+            self.metric['val'] = 0
+            self.metric['val_total'] = 0
+            self.metric['epoch_val_loss'] = []
+
+
+
+    def find_lr(self, optimizer):
+        if not self.is_find_max_lr:
+            return
+
+        lr_finder = LRFinder(self, optimizer, self.criterion)
+        lr_finder.range_test(self.train_dataloader(), end_lr=100, num_iter=100)
+        _, best_lr = lr_finder.plot()  # to inspect the loss-learning rate graph
+        lr_finder.reset()
+        self.max_lr = best_lr
+
+
+    def plot_model_performance(self):
+        fig, axs = plt.subplots(2, 2, figsize=(15, 10))
+        axs[0, 0].plot( self.metric['train_loss'] )
+        axs[0, 0].set_title("Training Loss")
+        axs[1, 0].plot( self.metric['train_acc'] )
+        axs[1, 0].set_title("Training Accuracy")
+        axs[0, 1].plot( self.metric['val_loss'] )
+        axs[0, 1].set_title("Test Loss")
+        axs[1, 1].plot( self.metric['val_acc'] )
+        axs[1, 1].set_title("Test Accuracy")
+
+
+    def plot_grad_cam(self, mean, std, target_layers, get_data_label_name, count=10, missclassified=True, grad_opacity=1.0):
+        cam = GradCAM(model=self, target_layers=target_layers)
+
+        #fig = plt.figure()
+        for i in range(count):
+            plt.subplot(int(count / 5), 5, i + 1)
+            plt.tight_layout()
+            if not missclassified:
+                pred_dict = test_correct_pred
+            else:
+                pred_dict = test_incorrect_pred
+
+            targets = [ClassifierOutputTarget(pred_dict['ground_truths'][i].cpu().item())]
+
+            grayscale_cam = cam(input_tensor=pred_dict['images'][i][None, :].cpu(), targets=targets)
+
+            x = denormalize(pred_dict['images'][i].cpu(), mean, std)
+
+            image = np.array(255 * x, np.int16).transpose(1, 2, 0)
+            img_tensor = np.array(x, np.float16).transpose(1, 2, 0)
+
+            visualization = show_cam_on_image(img_tensor, grayscale_cam.transpose(1, 2, 0), use_rgb=True,
+                                              image_weight=(1.0 - grad_opacity) )
+
+            plt.imshow(image, vmin=0, vmax=255)
+            plt.imshow(visualization, vmin=0, vmax=255, alpha=grad_opacity)
+            plt.xticks([])
+            plt.yticks([])
+
+            title = get_data_label_name(pred_dict['ground_truths'][i].item()) + ' / ' + \
+                    get_data_label_name(pred_dict['predicted_vals'][i].item())
+            plt.title(title, fontsize=8)

requirements.txt

@@ -0,0 +1,7 @@
+torch
+torch-lr-finder
+torchvision
+pillow
+gradio
+grad-cam
+numpy

utils.py

@@ -0,0 +1,62 @@
+import os
+import torch
+
+def get_dataset_labels():
+    return ['airplane','automobile','bird','cat','deer','dog','frog','horse','ship','truck']
+
+
+def get_data_label_name(idx):
+    if idx < 0:
+        return ''
+
+    return get_dataset_labels()[idx]
+
+
+def get_data_idx_from_name(name):
+    if not name:
+        return -1
+
+    return get_dataset_labels.index(name.lower()) if name.lower() in get_dataset_labels() else -1
+
+def load_model_from_checkpoint(device, file_name='checkpoint.ckpt'):
+    checkpoint = torch.load('ckpt.pth', map_location=device)
+
+    return checkpoint
+
+
+def denormalize(img, mean, std):
+    MEAN = torch.tensor(mean)
+    STD = torch.tensor(std)
+
+    img = img * STD[:, None, None] + MEAN[:, None, None]
+    i_min = img.min().item()
+    i_max = img.max().item()
+
+    img_bar = (img - i_min)/(i_max - i_min)
+
+    return img_bar
+
+# Data to plot accuracy and loss graphs
+train_losses = []
+test_losses = []
+train_acc = []
+test_acc = []
+
+test_incorrect_pred = {'images': [], 'ground_truths': [], 'predicted_vals': []}
+test_correct_pred = {'images': [], 'ground_truths': [], 'predicted_vals': []}
+
+def get_correct_pred_count(pPrediction, pLabels):
+    return pPrediction.argmax(dim=1).eq(pLabels).sum().item()
+
+
+def add_predictions(data, pred, target):
+    diff_preds = pred.argmax(dim=1) - target
+    for idx, d in enumerate(diff_preds):
+        if d.item() != 0:
+            test_incorrect_pred['images'].append(data[idx])
+            test_incorrect_pred['ground_truths'].append(target[idx])
+            test_incorrect_pred['predicted_vals'].append(pred.argmax(dim=1)[idx])
+        elif d.item() == 0:
+            test_correct_pred['images'].append(data[idx])
+            test_correct_pred['ground_truths'].append(target[idx])
+            test_correct_pred['predicted_vals'].append(pred.argmax(dim=1)[idx])