<reponame>ysethi92/NLP-Project-LU
# -*- coding: utf-8 -*-
"""bert-lstm-main.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1xBSBc5C1reQsbUo-ogrdXzCNJAVX1pbe
"""

#INSTALL PACKAGES
# pip install -qq plotly
# pip install -qq icecream
# pip install -qq transformers


#IMPORT PACKAGES
import torch
from transformers import AutoTokenizer, AutoModel
import pandas as pd
import numpy as np
import random as rn
from icecream import ic 
from torch.utils.data import Dataset, DataLoader
import os

# INITIALIZE GLOBAL VARIABLES
train_count = 8000 #8000  
MAX_SENTENCE_LENGTH = 128
MAX_SENTENCES = 5
MAX_LENGTH = 100
BASENET = 'bert-base-uncased'

# SELECT DEVICE CPU/CUDA
cuda_available = torch.cuda.is_available()
device   = torch.device("cuda") if (cuda_available) else torch.device("cpu")

seed=1
torch.manual_seed(seed)
if(cuda_available):
	torch.cuda.manual_seed(seed)
	torch.backends.cudnn.benchmark = True
np.random.seed(seed)

ic(torch.cuda.is_available(), device, seed)

from google.colab import drive
drive.mount('/content/drive')

# Commented out IPython magic to ensure Python compatibility.
# LOAD DATA
# !pwd
# %cd /content/drive/MyDrive/Colab\ Notebooks/Spring\ 21/NLP/Humour-Detection-2021

#DATA COLUMNS
train_df = pd.read_csv("data/train.csv")
display(train_df.count())
display(train_df[:1])
ic(train_df['text'][0])

#AVERAGE LOSS
class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

# CUSTOM DATASET CLASS
class MyCustomDataset(Dataset): #dataset from torch
    def __init__(self, filename, basenet= 'bert-base-uncased', max_len= 128):
        super(MyCustomDataset, self).__init__()
        self.data    = pd.read_csv(filename) #"data/train.csv"
        self.labels = self.data['is_humor']
        self.text = self.data['text']
        self.tokenizer = AutoTokenizer.from_pretrained(basenet)
        self.max_len = max_len
        self.segment_id = torch.tensor([1] * self.max_len).view(1, -1)

    def __len__(self):
        return len(self.labels)

    def __getitem__(self, idx):
        label = self.labels[idx]
        text = self.text[idx]

        label = torch.tensor(label)

        encoded_input = self.tokenizer(text= text, # sentence
							 add_special_tokens= True, # [CLS] [SEP]
							 max_length= self.max_len, 
							 padding= 'max_length', 
							 return_attention_mask= True,
							 return_tensors = 'pt',  # return torch tensors
							 truncation= True
							)
        item = {"input_id": encoded_input['input_ids'], 
                "attention_mask_id": encoded_input['attention_mask'],
                "segment_id": self.segment_id,
                "true_label": label
              }
        
        return item

#MODEL ARCHITECTURE CLASS
class MyClassifier(torch.nn.Module):

    def __init__(self, basenet='bert-base-uncased', n_outputs=2, n_hidden=256):
        super(MyClassifier, self).__init__()
        self.encoder = AutoModel.from_pretrained( basenet,  output_hidden_states = False) 
       
        self.lstm  = torch.nn.LSTM(input_size= 768, hidden_size= 256)
        # self.classifier = torch.nn.Linear(in_features= 256, out_features= 2) #single fc
        self.classifier = torch.nn.Sequential(                                 
                            torch.nn.Linear(in_features= 256, out_features= 128),
													  torch.nn.ReLU(),
													  torch.nn.Dropout(p= 0.2),
													  torch.nn.Linear(in_features= 128, out_features= n_outputs)
                          )

    def forward(self, input_id, attention_mask_id, token_type_id):
        
        embeddings = self.encoder(
            input_ids= input_id, 
            attention_mask= attention_mask_id, 
            token_type_ids= token_type_id)['last_hidden_state']
        
        # ic(embeddings)
        embeddings= embeddings.squeeze().permute(1, 0, 2)
        
        # ic(embeddings.shape) #torch.Size([128, 5, 768])
        # "out" will give you access to all hidden states in the sequence
        # "hidden" will allow you to continue the sequence and backpropagate,
        # by passing it as an argument  to the lstm at a later time
        # Add the extra 2nd dimension
        lstm_out, lstm_hidden = self.lstm(embeddings)
        # ic(lstm_hidden[0].shape)  #torch.Size([1, 5, 256])

        embeddings    = lstm_hidden[0].squeeze()
        out = self.classifier (embeddings)											
        # ic(out.shape)
        
        return out

# TRAINING FUNCTION
def train(model, 
          dl, 
          optimizer, 
          device,
          criterion = torch.nn.CrossEntropyLoss()
          ):
    model.train()
    loss_avg = AverageMeter()
    
    for _, batch in enumerate(dl): #iterate in batch
      # ic(batch['input_id'].shape) #[32, 1, 128] (batch size, [one sentence,max-length of input])
      targets = batch['true_label'].to(device)
      input_id   = batch['input_id'].to(device).squeeze()
      attention_mask_id  = batch['attention_mask_id'].to(device).squeeze()
      segment_id = batch['segment_id'].to(device).squeeze()

      outputs = model(input_id, attention_mask_id, segment_id) # [32, 2] [batch size, model n_outputs]
      t = targets.float().view(-1,1) #[32,1]

      # ic(outputs[:, 0:2].shape, t[:, 0].shape) #outputs - first 2 cols of all rows
      loss = criterion(outputs[:, 0:2], t[:, 0].long())
      optimizer.zero_grad()
      loss.backward()
      optimizer.step()
      loss_avg.update(loss.item(), targets.shape[0]) #[32,1] batch size
    
    return loss_avg.avg


# TESTING/EVALUATION FUNCTION

from sklearn.metrics import f1_score, accuracy_score

def test(model, 
         dl, 
         device, 
         criterion = torch.nn.CrossEntropyLoss()):
    model.eval()
    loss_avg = AverageMeter()
    o_   = torch.tensor([]).to(device) #use for metrics
    t_   = torch.tensor([]).to(device)

    with torch.no_grad():
        for _, batch in enumerate(dl):
            targets = batch['true_label'].to(device) #[128] batch size
            input_id   = batch['input_id'].to(device).squeeze()
            attention_mask_id  = batch['attention_mask_id'].to(device).squeeze()
            segment_id = batch['segment_id'].to(device).squeeze()

            outputs = model(input_id, attention_mask_id, segment_id)
            t = targets.float().view(-1,1) #[128,1] batch size
            loss = criterion(outputs[:, 0:2], t[:, 0].long())

            # EVALUATE
            max_o = torch.argmax(outputs[:, 0:2], dim=1).view(-1,1)
            # ic(max_o.shape) #[128,1]
            max_o = max_o.detach()
            o_    = torch.cat((o_, max_o)  , 0) #float and append
            t_    = torch.cat((t_, targets), 0)
            loss_avg.update(loss.item(), targets.shape[0])

    o_ = o_.cpu().numpy()
    t_ = t_.view(-1,1)# change shape same as o_
    t_ = t_.cpu().numpy()
    ic(t_.shape, o_.shape)
    fscore = f1_score(t_[:, 0], o_[:, 0])
    accuracy= accuracy_score(t_[:, 0], o_[:, 0])

    return loss_avg.avg, [fscore, accuracy]

# LOAD DATASET 
train_dl = torch.utils.data.DataLoader(dataset= MyCustomDataset(
                          filename=  "data/train.csv",
                          basenet= BASENET),
                           batch_size= 32, shuffle= True, num_workers= 32, pin_memory= True)
val_dl = torch.utils.data.DataLoader(dataset= MyCustomDataset(
                          filename=  "data/dev.csv",
                          basenet= BASENET),
                           batch_size= 128, shuffle= True, num_workers= 32, pin_memory= True)
test_dl = torch.utils.data.DataLoader(dataset= MyCustomDataset(
                          filename=  "data/gold-test-27446.csv",
                          basenet= BASENET),
                           batch_size= 128, shuffle= True, num_workers= 32, pin_memory= True)

# MODEL SELECTION
model = MyClassifier(BASENET, n_outputs=2) #experiment - bert with lstm

if torch.cuda.device_count() > 1:
	model = torch.nn.DataParallel(model)
 
model = model.to(device)

# START TRAINING
learning_rate=2e-5 
start_epoch = 1
epochs = 10
optimizer = torch.optim.Adam(model.parameters(), lr= learning_rate)
train_loss_arr = []
val_loss_arr = []
metrics_arr = [] #[f1 , accuracy]

for ep in range(start_epoch, start_epoch+epochs):
  train_loss = train(model, train_dl, optimizer, device)
  val_loss, metrics = test(model, val_dl, device)
  train_loss_arr.append(train_loss)
  val_loss_arr.append(val_loss)
  metrics_arr.append(metrics)
  ic()

# EVALUATE GOLD TEST
test_loss, test_metrics = test(model, test_dl, device)

# REPORT EXPERIMENT RESULTS
print('-'*60)
print('train loss: ', end =" ") 
print('%.4f'%train_loss_arr[-1])
print('-'*60)

print(' val loss: ', end =" ")
print('%.4f'%val_loss_arr[-1], end =" " )      
print(' F1 score: ', end =" ")
print('%.2f'%(metrics_arr[-1][0]*100), end =" " )
print(' Accuracy: ', end =" ")
print('%.2f'%(metrics_arr[-1][1]*100))
print('-'*60)

print('test loss: ', end =" ")
print('%.4f'%test_loss, end =" " )      
print(' F1 score: ', end =" ")
print('%.2f'%(test_metrics[0]*100), end =" " )
print(' Accuracy: ', end =" ")
print('%.2f'%(test_metrics[1]*100) )
print('-'*60)

# SAVE MODEL
run_iter = 6 #iterations you ran the model
save_dir = os.path.join('my_model_weights', 'run_{}'.format(run_iter))

if not os.path.exists(save_dir):
  print("Creating directory {}".format(save_dir))
  os.makedirs(save_dir)

save_path = os.path.join(save_dir, "epoch_{}".format(epochs) + ".pth")
state = {
			'model': model.cpu().state_dict(),
			'optimizer': optimizer.state_dict(),
			'epoch': epochs,
		}

torch.save(state, save_path)
model = model.to(device)

def load_model_weights(model, path= None):
    
    m = torch.load(path, map_location= torch.device('cpu'))
    if 'state_dict' in m:
      sd = m['state_dict']
    else:
      sd = m['model']

    model_dict = model.state_dict()

    state_dict = {k: v for k, v in sd.items() if (k in model_dict) and (v.shape == model_dict[k].shape)}
    model_dict.update(state_dict) 
    model.load_state_dict(model_dict)

    if len(model_dict)!=len(state_dict):
        ic()
        
    start_epoch = m['epoch'] + 1
    return model, start_epoch

# LOAD SAVED MODEL
#!pwd
# %cd /content/drive/MyDrive/Colab\ Notebooks/Spring\ 21/NLP/Humour-Detection-2021 #colab - cd to project directory which contains my_model_weights directory

#path = "my_model_weights/run_5/epoch_10.pth" #uncomment this for bert-lstm weights
#model = MyClassifier(BASENET, n_outputs=2)
#model, start_epoch = load_model_weights(model, path)
#ic(start_epoch)
#model = model.to(device)

# PLOT GRAPHS
import plotly.graph_objects as go

x_axis = list(range(1, len(train_loss_arr)+1)) #epochs
y_axis = list(train_loss_arr)

# loss per epoch
fig = go.Figure(data = go.Scatter(x= x_axis, y= y_axis))
fig.update_layout(title='train loss',
                xaxis_title='epochs',
                yaxis_title='loss')
fig.show()

# performance per epoch - f1 and accuracy score
y2_axis = np.array(metrics_arr)
fig2 = go.Figure()
fig2.add_trace(go.Scatter(x= x_axis, y= y2_axis[:,0],
                  mode='lines',
                  name='f1-score'))
fig2.add_trace(go.Scatter(x= x_axis, y= y2_axis[:,1],
                  mode='lines+markers',
                  name='accuracy'))
fig2.update_layout(title='performance',
                xaxis_title='epochs',
                yaxis_title='metrics')
fig2.show()

# -*- coding: utf-8 -*-
"""bert-lstm-main.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1xBSBc5C1reQsbUo-ogrdXzCNJAVX1pbe
"""

#INSTALL PACKAGES
# pip install -qq plotly
# pip install -qq icecream
# pip install -qq transformers


#IMPORT PACKAGES
import torch
from transformers import AutoTokenizer, AutoModel
import pandas as pd
import numpy as np
import random as rn
from icecream import ic 
from torch.utils.data import Dataset, DataLoader
import os

# INITIALIZE GLOBAL VARIABLES
train_count = 8000 #8000  
MAX_SENTENCE_LENGTH = 128
MAX_SENTENCES = 5
MAX_LENGTH = 100
BASENET = 'bert-base-uncased'

# SELECT DEVICE CPU/CUDA
cuda_available = torch.cuda.is_available()
device   = torch.device("cuda") if (cuda_available) else torch.device("cpu")

seed=1
torch.manual_seed(seed)
if(cuda_available):
	torch.cuda.manual_seed(seed)
	torch.backends.cudnn.benchmark = True
np.random.seed(seed)

ic(torch.cuda.is_available(), device, seed)

from google.colab import drive
drive.mount('/content/drive')

# Commented out IPython magic to ensure Python compatibility.
# LOAD DATA
# !pwd
# %cd /content/drive/MyDrive/Colab\ Notebooks/Spring\ 21/NLP/Humour-Detection-2021

#DATA COLUMNS
train_df = pd.read_csv("data/train.csv")
display(train_df.count())
display(train_df[:1])
ic(train_df['text'][0])

#AVERAGE LOSS
class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

# CUSTOM DATASET CLASS
class MyCustomDataset(Dataset): #dataset from torch
    def __init__(self, filename, basenet= 'bert-base-uncased', max_len= 128):
        super(MyCustomDataset, self).__init__()
        self.data    = pd.read_csv(filename) #"data/train.csv"
        self.labels = self.data['is_humor']
        self.text = self.data['text']
        self.tokenizer = AutoTokenizer.from_pretrained(basenet)
        self.max_len = max_len
        self.segment_id = torch.tensor([1] * self.max_len).view(1, -1)

    def __len__(self):
        return len(self.labels)

    def __getitem__(self, idx):
        label = self.labels[idx]
        text = self.text[idx]

        label = torch.tensor(label)

        encoded_input = self.tokenizer(text= text, # sentence
							 add_special_tokens= True, # [CLS] [SEP]
							 max_length= self.max_len, 
							 padding= 'max_length', 
							 return_attention_mask= True,
							 return_tensors = 'pt',  # return torch tensors
							 truncation= True
							)
        item = {"input_id": encoded_input['input_ids'], 
                "attention_mask_id": encoded_input['attention_mask'],
                "segment_id": self.segment_id,
                "true_label": label
              }
        
        return item

#MODEL ARCHITECTURE CLASS
class MyClassifier(torch.nn.Module):

    def __init__(self, basenet='bert-base-uncased', n_outputs=2, n_hidden=256):
        super(MyClassifier, self).__init__()
        self.encoder = AutoModel.from_pretrained( basenet,  output_hidden_states = False) 
       
        self.lstm  = torch.nn.LSTM(input_size= 768, hidden_size= 256)
        # self.classifier = torch.nn.Linear(in_features= 256, out_features= 2) #single fc
        self.classifier = torch.nn.Sequential(                                 
                            torch.nn.Linear(in_features= 256, out_features= 128),
													  torch.nn.ReLU(),
													  torch.nn.Dropout(p= 0.2),
													  torch.nn.Linear(in_features= 128, out_features= n_outputs)
                          )

    def forward(self, input_id, attention_mask_id, token_type_id):
        
        embeddings = self.encoder(
            input_ids= input_id, 
            attention_mask= attention_mask_id, 
            token_type_ids= token_type_id)['last_hidden_state']
        
        # ic(embeddings)
        embeddings= embeddings.squeeze().permute(1, 0, 2)
        
        # ic(embeddings.shape) #torch.Size([128, 5, 768])
        # "out" will give you access to all hidden states in the sequence
        # "hidden" will allow you to continue the sequence and backpropagate,
        # by passing it as an argument  to the lstm at a later time
        # Add the extra 2nd dimension
        lstm_out, lstm_hidden = self.lstm(embeddings)
        # ic(lstm_hidden[0].shape)  #torch.Size([1, 5, 256])

        embeddings    = lstm_hidden[0].squeeze()
        out = self.classifier (embeddings)											
        # ic(out.shape)
        
        return out

# TRAINING FUNCTION
def train(model, 
          dl, 
          optimizer, 
          device,
          criterion = torch.nn.CrossEntropyLoss()
          ):
    model.train()
    loss_avg = AverageMeter()
    
    for _, batch in enumerate(dl): #iterate in batch
      # ic(batch['input_id'].shape) #[32, 1, 128] (batch size, [one sentence,max-length of input])
      targets = batch['true_label'].to(device)
      input_id   = batch['input_id'].to(device).squeeze()
      attention_mask_id  = batch['attention_mask_id'].to(device).squeeze()
      segment_id = batch['segment_id'].to(device).squeeze()

      outputs = model(input_id, attention_mask_id, segment_id) # [32, 2] [batch size, model n_outputs]
      t = targets.float().view(-1,1) #[32,1]

      # ic(outputs[:, 0:2].shape, t[:, 0].shape) #outputs - first 2 cols of all rows
      loss = criterion(outputs[:, 0:2], t[:, 0].long())
      optimizer.zero_grad()
      loss.backward()
      optimizer.step()
      loss_avg.update(loss.item(), targets.shape[0]) #[32,1] batch size
    
    return loss_avg.avg


# TESTING/EVALUATION FUNCTION

from sklearn.metrics import f1_score, accuracy_score

def test(model, 
         dl, 
         device, 
         criterion = torch.nn.CrossEntropyLoss()):
    model.eval()
    loss_avg = AverageMeter()
    o_   = torch.tensor([]).to(device) #use for metrics
    t_   = torch.tensor([]).to(device)

    with torch.no_grad():
        for _, batch in enumerate(dl):
            targets = batch['true_label'].to(device) #[128] batch size
            input_id   = batch['input_id'].to(device).squeeze()
            attention_mask_id  = batch['attention_mask_id'].to(device).squeeze()
            segment_id = batch['segment_id'].to(device).squeeze()

            outputs = model(input_id, attention_mask_id, segment_id)
            t = targets.float().view(-1,1) #[128,1] batch size
            loss = criterion(outputs[:, 0:2], t[:, 0].long())

            # EVALUATE
            max_o = torch.argmax(outputs[:, 0:2], dim=1).view(-1,1)
            # ic(max_o.shape) #[128,1]
            max_o = max_o.detach()
            o_    = torch.cat((o_, max_o)  , 0) #float and append
            t_    = torch.cat((t_, targets), 0)
            loss_avg.update(loss.item(), targets.shape[0])

    o_ = o_.cpu().numpy()
    t_ = t_.view(-1,1)# change shape same as o_
    t_ = t_.cpu().numpy()
    ic(t_.shape, o_.shape)
    fscore = f1_score(t_[:, 0], o_[:, 0])
    accuracy= accuracy_score(t_[:, 0], o_[:, 0])

    return loss_avg.avg, [fscore, accuracy]

# LOAD DATASET 
train_dl = torch.utils.data.DataLoader(dataset= MyCustomDataset(
                          filename=  "data/train.csv",
                          basenet= BASENET),
                           batch_size= 32, shuffle= True, num_workers= 32, pin_memory= True)
val_dl = torch.utils.data.DataLoader(dataset= MyCustomDataset(
                          filename=  "data/dev.csv",
                          basenet= BASENET),
                           batch_size= 128, shuffle= True, num_workers= 32, pin_memory= True)
test_dl = torch.utils.data.DataLoader(dataset= MyCustomDataset(
                          filename=  "data/gold-test-27446.csv",
                          basenet= BASENET),
                           batch_size= 128, shuffle= True, num_workers= 32, pin_memory= True)

# MODEL SELECTION
model = MyClassifier(BASENET, n_outputs=2) #experiment - bert with lstm

if torch.cuda.device_count() > 1:
	model = torch.nn.DataParallel(model)
 
model = model.to(device)

# START TRAINING
learning_rate=2e-5 
start_epoch = 1
epochs = 10
optimizer = torch.optim.Adam(model.parameters(), lr= learning_rate)
train_loss_arr = []
val_loss_arr = []
metrics_arr = [] #[f1 , accuracy]

for ep in range(start_epoch, start_epoch+epochs):
  train_loss = train(model, train_dl, optimizer, device)
  val_loss, metrics = test(model, val_dl, device)
  train_loss_arr.append(train_loss)
  val_loss_arr.append(val_loss)
  metrics_arr.append(metrics)
  ic()

# EVALUATE GOLD TEST
test_loss, test_metrics = test(model, test_dl, device)

# REPORT EXPERIMENT RESULTS
print('-'*60)
print('train loss: ', end =" ") 
print('%.4f'%train_loss_arr[-1])
print('-'*60)

print(' val loss: ', end =" ")
print('%.4f'%val_loss_arr[-1], end =" " )      
print(' F1 score: ', end =" ")
print('%.2f'%(metrics_arr[-1][0]*100), end =" " )
print(' Accuracy: ', end =" ")
print('%.2f'%(metrics_arr[-1][1]*100))
print('-'*60)

print('test loss: ', end =" ")
print('%.4f'%test_loss, end =" " )      
print(' F1 score: ', end =" ")
print('%.2f'%(test_metrics[0]*100), end =" " )
print(' Accuracy: ', end =" ")
print('%.2f'%(test_metrics[1]*100) )
print('-'*60)

# SAVE MODEL
run_iter = 6 #iterations you ran the model
save_dir = os.path.join('my_model_weights', 'run_{}'.format(run_iter))

if not os.path.exists(save_dir):
  print("Creating directory {}".format(save_dir))
  os.makedirs(save_dir)

save_path = os.path.join(save_dir, "epoch_{}".format(epochs) + ".pth")
state = {
			'model': model.cpu().state_dict(),
			'optimizer': optimizer.state_dict(),
			'epoch': epochs,
		}

torch.save(state, save_path)
model = model.to(device)

def load_model_weights(model, path= None):
    
    m = torch.load(path, map_location= torch.device('cpu'))
    if 'state_dict' in m:
      sd = m['state_dict']
    else:
      sd = m['model']

    model_dict = model.state_dict()

    state_dict = {k: v for k, v in sd.items() if (k in model_dict) and (v.shape == model_dict[k].shape)}
    model_dict.update(state_dict) 
    model.load_state_dict(model_dict)

    if len(model_dict)!=len(state_dict):
        ic()
        
    start_epoch = m['epoch'] + 1
    return model, start_epoch

# LOAD SAVED MODEL
#!pwd
# %cd /content/drive/MyDrive/Colab\ Notebooks/Spring\ 21/NLP/Humour-Detection-2021 #colab - cd to project directory which contains my_model_weights directory

#path = "my_model_weights/run_5/epoch_10.pth" #uncomment this for bert-lstm weights
#model = MyClassifier(BASENET, n_outputs=2)
#model, start_epoch = load_model_weights(model, path)
#ic(start_epoch)
#model = model.to(device)

# PLOT GRAPHS
import plotly.graph_objects as go

x_axis = list(range(1, len(train_loss_arr)+1)) #epochs
y_axis = list(train_loss_arr)

# loss per epoch
fig = go.Figure(data = go.Scatter(x= x_axis, y= y_axis))
fig.update_layout(title='train loss',
                xaxis_title='epochs',
                yaxis_title='loss')
fig.show()

# performance per epoch - f1 and accuracy score
y2_axis = np.array(metrics_arr)
fig2 = go.Figure()
fig2.add_trace(go.Scatter(x= x_axis, y= y2_axis[:,0],
                  mode='lines',
                  name='f1-score'))
fig2.add_trace(go.Scatter(x= x_axis, y= y2_axis[:,1],
                  mode='lines+markers',
                  name='accuracy'))
fig2.update_layout(title='performance',
                xaxis_title='epochs',
                yaxis_title='metrics')
fig2.show()
# -*- coding: utf-8 -*- bert-lstm-main.ipynb  Automatically generated by Colaboratory.  Original file is located at     https://colab.research.google.com/drive/1xBSBc5C1reQsbUo-ogrdXzCNJAVX1pbe #INSTALL PACKAGES # pip install -qq plotly # pip install -qq icecream # pip install -qq transformers #IMPORT PACKAGES # INITIALIZE GLOBAL VARIABLES #8000 # SELECT DEVICE CPU/CUDA # Commented out IPython magic to ensure Python compatibility. # LOAD DATA # !pwd # %cd /content/drive/MyDrive/Colab\ Notebooks/Spring\ 21/NLP/Humour-Detection-2021 #DATA COLUMNS #AVERAGE LOSS Computes and stores the average and current value # CUSTOM DATASET CLASS #dataset from torch #"data/train.csv" # sentence # [CLS] [SEP] # return torch tensors #MODEL ARCHITECTURE CLASS # self.classifier = torch.nn.Linear(in_features= 256, out_features= 2) #single fc # ic(embeddings) # ic(embeddings.shape) #torch.Size([128, 5, 768]) # "out" will give you access to all hidden states in the sequence # "hidden" will allow you to continue the sequence and backpropagate, # by passing it as an argument  to the lstm at a later time # Add the extra 2nd dimension # ic(lstm_hidden[0].shape)  #torch.Size([1, 5, 256]) # ic(out.shape) # TRAINING FUNCTION #iterate in batch # ic(batch['input_id'].shape) #[32, 1, 128] (batch size, [one sentence,max-length of input]) # [32, 2] [batch size, model n_outputs] #[32,1] # ic(outputs[:, 0:2].shape, t[:, 0].shape) #outputs - first 2 cols of all rows #[32,1] batch size # TESTING/EVALUATION FUNCTION #use for metrics #[128] batch size #[128,1] batch size # EVALUATE # ic(max_o.shape) #[128,1] #float and append # change shape same as o_ # LOAD DATASET # MODEL SELECTION #experiment - bert with lstm # START TRAINING #[f1 , accuracy] # EVALUATE GOLD TEST # REPORT EXPERIMENT RESULTS # SAVE MODEL #iterations you ran the model # LOAD SAVED MODEL #!pwd # %cd /content/drive/MyDrive/Colab\ Notebooks/Spring\ 21/NLP/Humour-Detection-2021 #colab - cd to project directory which contains my_model_weights directory #path = "my_model_weights/run_5/epoch_10.pth" #uncomment this for bert-lstm weights #model = MyClassifier(BASENET, n_outputs=2) #model, start_epoch = load_model_weights(model, path) #ic(start_epoch) #model = model.to(device) # PLOT GRAPHS #epochs # loss per epoch # performance per epoch - f1 and accuracy score