# -*- coding: utf-8 -*-
"""risk_affinity_class.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1YW-8Ce6hiQ-rDlrRxG4pdIAB1swKip8x
"""

'''
This .py script takes parsed risk sentences and phrases from NASA programmatic 
documentation and uses a HuggingFace Zero-Shot classification piepeline to 
categorize the Risk Affinity into a Technical, Cost, Schedule, Programmatic
categorization. 
'''

#ZP - BEFORE STARTING - Go up to the options in Colab and select Runtime -> change runtime type -> hardware accelrator type (dropdown) -> (select) GPU. Then click save.

#ZP - This will install the packages in the Google Colab Runtime. Have to re-do each time you restart I think. Some are probably alread in Colab but I just threw it all in here.
!pip install transformers
!pip install torch
!pip install pandas
!pip install numpy
!pip install sklearn

# -*- coding: utf-8 -*-
"""
@author: <NAME> (truncated since I made this public to share with you)
"""

from transformers import pipeline
import torch
import pandas as pd
import os
import numpy as np
from sklearn.model_selection import train_test_split
from transformers import AutoModelForSequenceClassification, AutoTokenizer, RobertaForSequenceClassification
from transformers import TrainingArguments, RobertaConfig
from transformers import Trainer
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
from transformers import DistilBertTokenizerFast
from transformers import DistilBertForSequenceClassification
import matplotlib.pyplot as plt
from torch import nn
from sklearn.utils import compute_class_weight





# Set directory to folder containing Risk Data
risk_data_dir = './Risk Data/'

# Get list of files
risk_files = os.listdir(risk_data_dir)

# Remove any files that are not .csv
risk_files = [f for f in risk_files if f[-4:] == '.csv']

# Load files into DataFrame 

risk_df = pd.read_csv(risk_data_dir + risk_files[0])

for i in range(1, len(risk_files)):
    print('Appending file {} to dataframe...'.format(i))
    df = pd.read_csv(risk_data_dir + risk_files[i])
    risk_df = risk_df.append(df)

# Select only risk labels
risk_df = risk_df[risk_df['Risk_Affinity'].notnull()]

# Convert NaN to 0
risk_df = risk_df.fillna(0)

# Get label counts
print('Risk Affinity label counts: \n {}'.format(risk_df['Risk_Affinity'].value_counts()))

# Convert labels to category codes (numeric values)
risk_df['Risk_Affinity'] = risk_df['Risk_Affinity'].astype('category')

risk_df['Risk_Affinity_Cat'] = risk_df['Risk_Affinity'].cat.codes

# Get array of labels
labels = risk_df['Risk_Affinity'].to_numpy()

risk_df.head()

#ZP - Trying a smaller model. Seems to be faster than the default ~facebook large mnli.
#ZP - Added device=0 which uses GPU
## NOTE: If no GPU change device = 1
classifier = pipeline("zero-shot-classification", device=0, multi_class=True)


# Label to be predicted by classification
candidate_label = ['Cost', 'Schedule', 'Technical', 'Programmatic']

pred_labels = []

#ZP - As of 1/31/2022, added an argument to the the classifier declaration that uses GPU. The bit below should be done in batches to make it faster.
for i in range(risk_df.shape[0]):
    token = risk_df.iloc[i, 0]
    classifier_result = classifier(token, candidate_label)
    ind = np.argmax(classifier_result['scores'])
    pred_labels.append(candidate_label[ind])

# Get accuracy
zeroshot_acc = np.sum(np.array(pred_labels)== risk_df['Risk_Affinity']) / labels.size
print('Zero Shot Classification Accuracy: {}'.format(zeroshot_acc))

# Confusion matrix
zero_cm = confusion_matrix(labels, pred_labels)
disp = ConfusionMatrixDisplay(confusion_matrix=zero_cm)
disp.plot()

#ZP - disp show was throwing an error in colab so I commented it out
#disp.show()

def split_encode(df, x_col, y_col, tokenizer, val_pct = 0.15, test_pct = 0.15, seed = 42):
  '''
  Splits data into training, validation and test sets, and then encodes them.
  '''
  
  train_val, test = train_test_split(df[[x_col, y_col]], test_size=test_pct, random_state=seed, shuffle=True)
  train, val = train_test_split(train_val, test_size = val_pct, random_state=seed, shuffle=True)
  
  

  train_encodings = tokenizer(train[x_col].to_list(), truncation=True, padding=True
                              , max_length=512)
  val_encodings = tokenizer(val[x_col].to_list(), truncation=True, padding=True
                            , max_length=512)
  test_encodings = tokenizer(test[x_col].to_list(), truncation=True, padding=True
                             , max_length=512)

  train_labels = train[y_col].astype(int).to_list()
  val_labels = val[y_col].astype(int).to_list()
  test_labels = test[y_col].astype(int).to_list()  

  return ((train_encodings, train_labels), (val_encodings, val_labels), 
          (test_encodings, test_labels))

# Convert data to Torch dataset
# https://huggingface.co/transformers/v3.1.0/custom_datasets.html

class RiskDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

# Compute the class weights, which will be used to rebalance the data
# This will be feed into the compute loss function into the CustomTrainer subclass
class_wts = compute_class_weight(class_weight = 'balanced', 
                                 classes = np.unique(labels), 
                                 y = labels)

# Implement a custom trainer to use a weighted loss to account for 
# unbalananced data

class CustomTrainer(Trainer):
    def compute_loss(self, model, inputs, return_outputs=False):
        labels = inputs.get("labels")
        # forward pass
        outputs = model(**inputs)
        logits = outputs.get("logits")
        # compute custom loss (suppose one has 3 labels with different weights)
        loss_fct = nn.CrossEntropyLoss(weight=torch.tensor(class_wts).float()).to(device='cuda')
        loss = loss_fct(logits.view(-1, self.model.config.num_labels), labels.view(-1))
        return (loss, outputs) if return_outputs else loss

torch.cuda.empty_cache()

# Get dataset tuples using Distilbert Tokenizer
# https://huggingface.co/distilbert-base-uncased
distil_tokenizer = AutoTokenizer.from_pretrained('microsoft/deberta-base-mnli')
dat_tuple = split_encode(risk_df, 'Sentences', 'Risk_Affinity_Cat', distil_tokenizer)

# Convert data to torch dataset

train_dataset = RiskDataset(dat_tuple[0][0], dat_tuple[0][1])
val_dataset = RiskDataset(dat_tuple[1][0], dat_tuple[1][1])
test_dataset = RiskDataset(dat_tuple[2][0], dat_tuple[2][1])

test_labels = dat_tuple[2][1]


# Set up model


model = AutoModelForSequenceClassification.from_pretrained('microsoft/deberta-base-mnli', num_labels=len(class_wts), ignore_mismatched_sizes=True)

# Reduced batch size to ensure enough memory on the GPU in GoogleCollab
training_args = TrainingArguments("test_trainer", per_device_train_batch_size = 4)


trainer = CustomTrainer(model=model, args=training_args, train_dataset=train_dataset, eval_dataset=val_dataset)

trainer.train()

# Predict training data set
# https://huggingface.co/course/chapter3/3?fw=pt

pred = trainer.predict(test_dataset)
preds = np.argmax(pred.predictions, axis=-1)

test_acc = np.sum(preds == test_labels) / len(test_labels)

print('Test Accuracy: {}'.format(test_acc*100))

# Confusion matrix

cm = confusion_matrix(test_labels, preds)
disp_cm = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels = risk_df['Risk_Affinity'].cat.categories)
disp_cm.plot()
#disp_cm.show()

cm = confusion_matrix(test_labels, preds)
disp_cm = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels = risk_df['Risk_Affinity'].cat.categories)
disp_cm.plot()
#disp_cm.show()


# -*- coding: utf-8 -*-
"""risk_affinity_class.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1YW-8Ce6hiQ-rDlrRxG4pdIAB1swKip8x
"""

'''
This .py script takes parsed risk sentences and phrases from NASA programmatic 
documentation and uses a HuggingFace Zero-Shot classification piepeline to 
categorize the Risk Affinity into a Technical, Cost, Schedule, Programmatic
categorization. 
'''

#ZP - BEFORE STARTING - Go up to the options in Colab and select Runtime -> change runtime type -> hardware accelrator type (dropdown) -> (select) GPU. Then click save.

#ZP - This will install the packages in the Google Colab Runtime. Have to re-do each time you restart I think. Some are probably alread in Colab but I just threw it all in here.
!pip install transformers
!pip install torch
!pip install pandas
!pip install numpy
!pip install sklearn

# -*- coding: utf-8 -*-
"""
@author: <NAME> (truncated since I made this public to share with you)
"""

from transformers import pipeline
import torch
import pandas as pd
import os
import numpy as np
from sklearn.model_selection import train_test_split
from transformers import AutoModelForSequenceClassification, AutoTokenizer, RobertaForSequenceClassification
from transformers import TrainingArguments, RobertaConfig
from transformers import Trainer
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
from transformers import DistilBertTokenizerFast
from transformers import DistilBertForSequenceClassification
import matplotlib.pyplot as plt
from torch import nn
from sklearn.utils import compute_class_weight





# Set directory to folder containing Risk Data
risk_data_dir = './Risk Data/'

# Get list of files
risk_files = os.listdir(risk_data_dir)

# Remove any files that are not .csv
risk_files = [f for f in risk_files if f[-4:] == '.csv']

# Load files into DataFrame 

risk_df = pd.read_csv(risk_data_dir + risk_files[0])

for i in range(1, len(risk_files)):
    print('Appending file {} to dataframe...'.format(i))
    df = pd.read_csv(risk_data_dir + risk_files[i])
    risk_df = risk_df.append(df)

# Select only risk labels
risk_df = risk_df[risk_df['Risk_Affinity'].notnull()]

# Convert NaN to 0
risk_df = risk_df.fillna(0)

# Get label counts
print('Risk Affinity label counts: \n {}'.format(risk_df['Risk_Affinity'].value_counts()))

# Convert labels to category codes (numeric values)
risk_df['Risk_Affinity'] = risk_df['Risk_Affinity'].astype('category')

risk_df['Risk_Affinity_Cat'] = risk_df['Risk_Affinity'].cat.codes

# Get array of labels
labels = risk_df['Risk_Affinity'].to_numpy()

risk_df.head()

#ZP - Trying a smaller model. Seems to be faster than the default ~facebook large mnli.
#ZP - Added device=0 which uses GPU
## NOTE: If no GPU change device = 1
classifier = pipeline("zero-shot-classification", device=0, multi_class=True)


# Label to be predicted by classification
candidate_label = ['Cost', 'Schedule', 'Technical', 'Programmatic']

pred_labels = []

#ZP - As of 1/31/2022, added an argument to the the classifier declaration that uses GPU. The bit below should be done in batches to make it faster.
for i in range(risk_df.shape[0]):
    token = risk_df.iloc[i, 0]
    classifier_result = classifier(token, candidate_label)
    ind = np.argmax(classifier_result['scores'])
    pred_labels.append(candidate_label[ind])

# Get accuracy
zeroshot_acc = np.sum(np.array(pred_labels)== risk_df['Risk_Affinity']) / labels.size
print('Zero Shot Classification Accuracy: {}'.format(zeroshot_acc))

# Confusion matrix
zero_cm = confusion_matrix(labels, pred_labels)
disp = ConfusionMatrixDisplay(confusion_matrix=zero_cm)
disp.plot()

#ZP - disp show was throwing an error in colab so I commented it out
#disp.show()

def split_encode(df, x_col, y_col, tokenizer, val_pct = 0.15, test_pct = 0.15, seed = 42):
  '''
  Splits data into training, validation and test sets, and then encodes them.
  '''
  
  train_val, test = train_test_split(df[[x_col, y_col]], test_size=test_pct, random_state=seed, shuffle=True)
  train, val = train_test_split(train_val, test_size = val_pct, random_state=seed, shuffle=True)
  
  

  train_encodings = tokenizer(train[x_col].to_list(), truncation=True, padding=True
                              , max_length=512)
  val_encodings = tokenizer(val[x_col].to_list(), truncation=True, padding=True
                            , max_length=512)
  test_encodings = tokenizer(test[x_col].to_list(), truncation=True, padding=True
                             , max_length=512)

  train_labels = train[y_col].astype(int).to_list()
  val_labels = val[y_col].astype(int).to_list()
  test_labels = test[y_col].astype(int).to_list()  

  return ((train_encodings, train_labels), (val_encodings, val_labels), 
          (test_encodings, test_labels))

# Convert data to Torch dataset
# https://huggingface.co/transformers/v3.1.0/custom_datasets.html

class RiskDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

# Compute the class weights, which will be used to rebalance the data
# This will be feed into the compute loss function into the CustomTrainer subclass
class_wts = compute_class_weight(class_weight = 'balanced', 
                                 classes = np.unique(labels), 
                                 y = labels)

# Implement a custom trainer to use a weighted loss to account for 
# unbalananced data

class CustomTrainer(Trainer):
    def compute_loss(self, model, inputs, return_outputs=False):
        labels = inputs.get("labels")
        # forward pass
        outputs = model(**inputs)
        logits = outputs.get("logits")
        # compute custom loss (suppose one has 3 labels with different weights)
        loss_fct = nn.CrossEntropyLoss(weight=torch.tensor(class_wts).float()).to(device='cuda')
        loss = loss_fct(logits.view(-1, self.model.config.num_labels), labels.view(-1))
        return (loss, outputs) if return_outputs else loss

torch.cuda.empty_cache()

# Get dataset tuples using Distilbert Tokenizer
# https://huggingface.co/distilbert-base-uncased
distil_tokenizer = AutoTokenizer.from_pretrained('microsoft/deberta-base-mnli')
dat_tuple = split_encode(risk_df, 'Sentences', 'Risk_Affinity_Cat', distil_tokenizer)

# Convert data to torch dataset

train_dataset = RiskDataset(dat_tuple[0][0], dat_tuple[0][1])
val_dataset = RiskDataset(dat_tuple[1][0], dat_tuple[1][1])
test_dataset = RiskDataset(dat_tuple[2][0], dat_tuple[2][1])

test_labels = dat_tuple[2][1]


# Set up model


model = AutoModelForSequenceClassification.from_pretrained('microsoft/deberta-base-mnli', num_labels=len(class_wts), ignore_mismatched_sizes=True)

# Reduced batch size to ensure enough memory on the GPU in GoogleCollab
training_args = TrainingArguments("test_trainer", per_device_train_batch_size = 4)


trainer = CustomTrainer(model=model, args=training_args, train_dataset=train_dataset, eval_dataset=val_dataset)

trainer.train()

# Predict training data set
# https://huggingface.co/course/chapter3/3?fw=pt

pred = trainer.predict(test_dataset)
preds = np.argmax(pred.predictions, axis=-1)

test_acc = np.sum(preds == test_labels) / len(test_labels)

print('Test Accuracy: {}'.format(test_acc*100))

# Confusion matrix

cm = confusion_matrix(test_labels, preds)
disp_cm = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels = risk_df['Risk_Affinity'].cat.categories)
disp_cm.plot()
#disp_cm.show()

cm = confusion_matrix(test_labels, preds)
disp_cm = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels = risk_df['Risk_Affinity'].cat.categories)
disp_cm.plot()
#disp_cm.show()


# -*- coding: utf-8 -*- risk_affinity_class.ipynb  Automatically generated by Colaboratory.  Original file is located at     https://colab.research.google.com/drive/1YW-8Ce6hiQ-rDlrRxG4pdIAB1swKip8x This .py script takes parsed risk sentences and phrases from NASA programmatic  documentation and uses a HuggingFace Zero-Shot classification piepeline to  categorize the Risk Affinity into a Technical, Cost, Schedule, Programmatic categorization. #ZP - BEFORE STARTING - Go up to the options in Colab and select Runtime -> change runtime type -> hardware accelrator type (dropdown) -> (select) GPU. Then click save. #ZP - This will install the packages in the Google Colab Runtime. Have to re-do each time you restart I think. Some are probably alread in Colab but I just threw it all in here. # -*- coding: utf-8 -*- @author: <NAME> (truncated since I made this public to share with you) # Set directory to folder containing Risk Data # Get list of files # Remove any files that are not .csv # Load files into DataFrame # Select only risk labels # Convert NaN to 0 # Get label counts # Convert labels to category codes (numeric values) # Get array of labels #ZP - Trying a smaller model. Seems to be faster than the default ~facebook large mnli. #ZP - Added device=0 which uses GPU ## NOTE: If no GPU change device = 1 # Label to be predicted by classification #ZP - As of 1/31/2022, added an argument to the the classifier declaration that uses GPU. The bit below should be done in batches to make it faster. # Get accuracy # Confusion matrix #ZP - disp show was throwing an error in colab so I commented it out #disp.show() Splits data into training, validation and test sets, and then encodes them. # Convert data to Torch dataset # https://huggingface.co/transformers/v3.1.0/custom_datasets.html # Compute the class weights, which will be used to rebalance the data # This will be feed into the compute loss function into the CustomTrainer subclass # Implement a custom trainer to use a weighted loss to account for # unbalananced data # forward pass # compute custom loss (suppose one has 3 labels with different weights) # Get dataset tuples using Distilbert Tokenizer # https://huggingface.co/distilbert-base-uncased # Convert data to torch dataset # Set up model # Reduced batch size to ensure enough memory on the GPU in GoogleCollab # Predict training data set # https://huggingface.co/course/chapter3/3?fw=pt # Confusion matrix #disp_cm.show() #disp_cm.show()