removed head

app.py

@@ -101,24 +101,22 @@ import shutil
 import matplotlib.pyplot as plt
 from sklearn.metrics import roc_curve, auc
 # Define the function to process the input file and model selection
-<<<<<<< HEAD
+
 def process_file(file,label,info, model_name):
-=======
-def process_file(file,label, model_name):
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
+
     with open(file.name, 'r') as f:
         content = f.read()
     saved_test_dataset = "train.txt"
     saved_test_label = "train_label.txt"
-<<<<<<< HEAD
+
     saved_train_info="train_info.txt"
-=======
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
+
+
     
     # Save the uploaded file content to a specified location
     shutil.copyfile(file.name, saved_test_dataset)
     shutil.copyfile(label.name, saved_test_label)
-<<<<<<< HEAD
+
     shutil.copyfile(info.name, saved_train_info)
     # For demonstration purposes, we'll just return the content with the selected model name
     # if(model_name=="highGRschool10"):
@@ -142,7 +140,7 @@ def process_file(file,label, model_name):
         "-e",str(1),
         "-b",str(5)
     ], shell=True)
-=======
+
     # For demonstration purposes, we'll just return the content with the selected model name
     if(model_name=="FS"):
         checkpoint="ratio_proportion_change3/output/FS/bert_fine_tuned.model.ep32"
@@ -159,7 +157,7 @@ def process_file(file,label, model_name):
     subprocess.run(["python", "src/test_saved_model.py",
                     "--finetuned_bert_checkpoint",checkpoint
                     ])
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
+
     result = {}
     with open("result.txt", 'r') as file:
         for line in file:
@@ -194,11 +192,9 @@ def process_file(file,label, model_name):
     return text_output,plot_path
 
 # List of models for the dropdown menu
-<<<<<<< HEAD
+
 models = ["highGRschool10", "lowGRschoolAll", "fullTest"]
-=======
-models = ["FS", "IS", "CORRECTNESS","EFFECTIVENESS"]
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
+
 
 # Create the Gradio interface
 with gr.Blocks(css="""
@@ -388,25 +384,22 @@ tbody.svelte-18wv37q>tr.svelte-18wv37q:nth-child(odd) {
     with gr.Row():
         file_input = gr.File(label="Upload a test file", file_types=['.txt'], elem_classes="file-box")
         label_input = gr.File(label="Upload test labels", file_types=['.txt'], elem_classes="file-box")
-<<<<<<< HEAD
+
         info_input = gr.File(label="Upload test info", file_types=['.txt'], elem_classes="file-box")
     
     model_dropdown = gr.Dropdown(choices=models, label="Select Finetune Task", elem_classes="dropdown-menu")
-=======
+
     
-    model_dropdown = gr.Dropdown(choices=models, label="Select Model", elem_classes="dropdown-menu")
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
+
     
     with gr.Row():
         output_text = gr.Textbox(label="Output Text")
         output_image = gr.Image(label="Output Plot")
 
     btn = gr.Button("Submit")
-<<<<<<< HEAD
+
     btn.click(fn=process_file, inputs=[file_input,label_input,info_input, model_dropdown], outputs=[output_text,output_image])
-=======
-    btn.click(fn=process_file, inputs=[file_input,label_input, model_dropdown], outputs=[output_text,output_image])
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
+
 
 # Launch the app
 demo.launch()

src/attention.py

@@ -3,19 +3,11 @@ import torch.nn.functional as F
 import torch
 
 import math
-<<<<<<< HEAD
 import pickle
 
 class Attention(nn.Module):
     """
     Compute Scaled Dot Product Attention
-=======
-
-
-class Attention(nn.Module):
-    """
-    Compute 'Scaled Dot Product Attention
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
     """
     
     def __init__(self):
@@ -53,10 +45,6 @@ class MultiHeadedAttention(nn.Module):
         self.linear_layers = nn.ModuleList([nn.Linear(d_model, d_model) for _ in range(3)])
         self.output_linear = nn.Linear(d_model, d_model)
         self.attention = Attention()
-<<<<<<< HEAD
-=======
-
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         self.dropout = nn.Dropout(p=dropout)
 
     def forward(self, query, key, value, mask=None):
@@ -70,21 +58,9 @@ class MultiHeadedAttention(nn.Module):
         query, key, value = [l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)
                             for l, x in zip(self.linear_layers, (query, key, value))]
         # 2) Apply attention on all the projected vectors in batch. 
-<<<<<<< HEAD
         x, p_attn = self.attention(query, key, value, mask=mask, dropout=self.dropout)
                     
         # 3) "Concat" using a view and apply a final linear. 
         x = x.transpose(1, 2).contiguous().view(nbatches, -1, self.h * self.d_k)
         
         return self.output_linear(x), p_attn
-=======
-        x, attn = self.attention(query, key, value, mask=mask, dropout=self.dropout)
-        # torch.Size([64, 8, 100, 100])
-        # print("Attention", attn.shape)
-        
-        # 3) "Concat" using a view and apply a final linear. 
-        x = x.transpose(1, 2).contiguous().view(nbatches, -1, self.h * self.d_k)
-        
-        return self.output_linear(x)
-    
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896

src/bert.py

@@ -1,14 +1,8 @@
 import torch.nn as nn
-<<<<<<< HEAD
 import torch
 
 from .transformer import TransformerBlock
 from .embedding import BERTEmbedding
-=======
-
-from transformer import TransformerBlock
-from embedding import BERTEmbedding
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
 
 class BERT(nn.Module):
     """
@@ -38,15 +32,11 @@ class BERT(nn.Module):
         # multi-layers transformer blocks, deep network
         self.transformer_blocks = nn.ModuleList(
             [TransformerBlock(hidden, attn_heads, hidden * 4, dropout) for _ in range(n_layers)])
-<<<<<<< HEAD
         # self.attention_values = []
-=======
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
 
     def forward(self, x, segment_info):
         # attention masking for padded token
         # torch.ByteTensor([batch_size, 1, seq_len, seq_len)
-<<<<<<< HEAD
        
         device = x.device
         
@@ -68,15 +58,5 @@ class BERT(nn.Module):
         for transformer in self.transformer_blocks:
             x = transformer.forward(x, mask)
             # self.attention_values.append(transformer.p_attn)
-=======
-        mask = (x > 0).unsqueeze(1).repeat(1, x.size(1), 1).unsqueeze(1)
-        # print("bert mask: ", mask)
-        # embedding the indexed sequence to sequence of vectors
-        x = self.embedding(x, segment_info)
-
-        # running over multiple transformer blocks
-        for transformer in self.transformer_blocks:
-            x = transformer.forward(x, mask)
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
             
         return x

src/classifier_model.py

@@ -1,28 +1,17 @@
-<<<<<<< HEAD
 import torch
 import torch.nn as nn
 
 from .bert import BERT
-=======
-import torch.nn as nn
-
-from bert import BERT
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
 
 
 class BERTForClassification(nn.Module):
     """
-<<<<<<< HEAD
         Fine-tune Task Classifier Model
-=======
-        Progress Classifier Model
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
     """
 
     def __init__(self, bert: BERT, vocab_size, n_labels):
         """
         :param bert: BERT model which should be trained
-<<<<<<< HEAD
         :param vocab_size: total vocab size
         :param n_labels: number of labels for the task
         """
@@ -59,17 +48,4 @@ class BERTForClassificationWithFeats(nn.Module):
         # x = self.linear1(x)
         # x = self.RELU(x)
         # return self.linear2(x)
-        return self.linear(x)
-=======
-        :param vocab_size: total vocab size for masked_lm
-        """
-
-        super().__init__()
-        self.bert = bert
-        self.linear = nn.Linear(self.bert.hidden, n_labels)
-        # self.softmax = nn.LogSoftmax(dim=-1)
-
-    def forward(self, x, segment_label):
-        x = self.bert(x, segment_label)
-        return x, self.linear(x[:, 0])
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
+        return self.linear(x)

src/dataset.py

@@ -4,28 +4,17 @@ import pandas as pd
 import numpy as np
 import tqdm
 import random
-<<<<<<< HEAD
 from .vocab import Vocab
 import pickle
 import copy
 # from sklearn.preprocessing import OneHotEncoder
-=======
-from vocab import Vocab
-import pickle
-import copy
-from sklearn.preprocessing import OneHotEncoder
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
 
 class PretrainerDataset(Dataset):
     """
         Class name: PretrainDataset
         
     """
-<<<<<<< HEAD
     def __init__(self, dataset_path, vocab, seq_len=30, max_mask=0.15):
-=======
-    def __init__(self, dataset_path, vocab, seq_len=30, select_next_seq= False):
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         self.dataset_path = dataset_path
         self.vocab = vocab # Vocab object
         
@@ -46,7 +35,6 @@ class PretrainerDataset(Dataset):
                         self.index_documents[i] = []
                     else:
                         self.index_documents[i].append(index)
-<<<<<<< HEAD
                         self.lines.append(line.split("\t"))
                         len_line = len(line.split("\t"))
                         seq_len_list.append(len_line)
@@ -61,22 +49,6 @@ class PretrainerDataset(Dataset):
         print("Sequence length set at: ", self.seq_len)
         self.max_mask = max_mask
         print("% of input tokens selected for masking : ",self.max_mask)
-=======
-                        self.lines.append(line.split())
-                        len_line = len(line.split())
-                        seq_len_list.append(len_line)
-                        index+=1
-            reader.close()
-        print("Sequence Stats: ", len(seq_len_list), min(seq_len_list), max(seq_len_list), sum(seq_len_list)/len(seq_len_list))
-        print("Unique Sequences: ", len({tuple(ll) for ll in self.lines}))
-        self.index_documents = {k:v for k,v in self.index_documents.items() if v}
-        self.seq_len = seq_len
-        self.max_mask_per_seq = 0.15
-        self.select_next_seq = select_next_seq
-        print("Sequence length set at ", self.seq_len)
-        print("select_next_seq: ", self.select_next_seq)
-        print(len(self.index_documents))
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
 
     
     def __len__(self):
@@ -84,7 +56,6 @@ class PretrainerDataset(Dataset):
     
     def __getitem__(self, item):
         token_a = self.lines[item]
-<<<<<<< HEAD
         # sa_masked = None
         # sa_masked_label = None
         # token_b = None
@@ -130,44 +101,6 @@ class PretrainerDataset(Dataset):
         
         # print(item, len(s1), len(s1_label), len(segment_label))
         # print(f"{item}.")
-=======
-        token_b = None
-        is_same_student = None
-        sa_masked = None
-        sa_masked_label = None
-        sb_masked = None
-        sb_masked_label = None
-        
-        if self.select_next_seq:
-            is_same_student, token_b = self.get_token_b(item)
-            is_same_student = 1 if is_same_student else 0
-            token_a1, token_b1 = self.truncate_to_max_seq(token_a, token_b)
-            sa_masked, sa_masked_label = self.random_mask_seq(token_a1)
-            sb_masked, sb_masked_label = self.random_mask_seq(token_b1)
-        else:
-            token_a = token_a[:self.seq_len-2]
-            sa_masked, sa_masked_label = self.random_mask_seq(token_a)
-        
-        s1 = ([self.vocab.vocab['[CLS]']] + sa_masked + [self.vocab.vocab['[SEP]']])
-        s1_label = ([self.vocab.vocab['[PAD]']] + sa_masked_label + [self.vocab.vocab['[PAD]']])
-        segment_label = [1 for _ in range(len(s1))]
-        
-        if self.select_next_seq:
-            s1 = s1 + sb_masked + [self.vocab.vocab['[SEP]']]
-            s1_label = s1_label + sb_masked_label + [self.vocab.vocab['[PAD]']]
-            segment_label = segment_label + [2 for _ in range(len(sb_masked)+1)]
-        
-        padding = [self.vocab.vocab['[PAD]'] for _ in range(self.seq_len - len(s1))]
-        s1.extend(padding), s1_label.extend(padding), segment_label.extend(padding)
- 
-        output = {'bert_input': s1,
-                 'bert_label': s1_label,
-                 'segment_label': segment_label}
-        
-        if self.select_next_seq:
-            output['is_same_student'] = is_same_student
-        # print(item, len(s1), len(s1_label), len(segment_label))
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         return {key: torch.tensor(value) for key, value in output.items()}
     
     def random_mask_seq(self, tokens):
@@ -176,7 +109,6 @@ class PretrainerDataset(Dataset):
         Output: masked token seq, output label
         """
         
-<<<<<<< HEAD
         masked_pos = []
         output_labels = []
         output_tokens = copy.deepcopy(tokens)
@@ -197,22 +129,11 @@ class PretrainerDataset(Dataset):
             # else:    
             prob = random.random()
             if prob < self.max_mask:
-=======
-        # masked_pos_label = {}
-        output_labels = []
-        output_tokens = copy.deepcopy(tokens)
-        
-        # while(len(label_tokens) < self.max_mask_per_seq*len(tokens)):
-        for i, token in enumerate(tokens):
-            prob = random.random()
-            if prob < 0.15:
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
              # chooses 15% of token positions at random
                 # prob /= 0.15
                 prob = random.random()
                 if prob < 0.8: #[MASK] token 80% of the time
                     output_tokens[i] = self.vocab.vocab['[MASK]']
-<<<<<<< HEAD
                     masked_pos.append(1)
                 elif prob < 0.9: # a random token 10% of the time 
                     # print(".......0.8-0.9......")
@@ -226,14 +147,6 @@ class PretrainerDataset(Dataset):
                     # print(".......unchanged......")
                     output_tokens[i] = self.vocab.vocab.get(token, self.vocab.vocab['[UNK]'])
                     masked_pos.append(0)
-=======
-                elif prob < 0.9: # a random token 10% of the time 
-                    # print(".......0.8-0.9......")
-                    output_tokens[i] = random.randint(1, len(self.vocab.vocab)-1)
-                else: # the unchanged i-th token 10% of the time
-                    # print(".......unchanged......")
-                    output_tokens[i] = self.vocab.vocab.get(token, self.vocab.vocab['[UNK]'])
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
                 # True Label
                 output_labels.append(self.vocab.vocab.get(token, self.vocab.vocab['[UNK]']))
                 # masked_pos_label[i] = self.vocab.vocab.get(token, self.vocab.vocab['[UNK]'])
@@ -242,16 +155,12 @@ class PretrainerDataset(Dataset):
                 output_tokens[i] = self.vocab.vocab.get(token, self.vocab.vocab['[UNK]'])
                 # Padded label
                 output_labels.append(self.vocab.vocab['[PAD]'])
-<<<<<<< HEAD
                 masked_pos.append(0)
-=======
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         # label_position = []
         # label_tokens = []
         # for k, v in masked_pos_label.items():
         #     label_position.append(k)
         #     label_tokens.append(v)
-<<<<<<< HEAD
         return  output_tokens, output_labels, masked_pos
     
 #     def get_token_b(self, item):
@@ -288,43 +197,6 @@ class PretrainerDataset(Dataset):
 #                 sb.pop()
 #         return sa, sb
             
-=======
-        return  output_tokens, output_labels
-    
-    def get_token_b(self, item):
-        document_id = [k for k,v in self.index_documents.items() if item in v][0]
-        random_document_id = document_id
-        
-        if random.random() < 0.5:
-            document_ids = [k for k in self.index_documents.keys() if k != document_id]
-            random_document_id = random.choice(document_ids) 
-
-        same_student = (random_document_id == document_id)
-        
-        nex_seq_list = self.index_documents.get(random_document_id)
-
-        if same_student:
-            if len(nex_seq_list) != 1:
-                nex_seq_list = [v for v in nex_seq_list if v !=item]
-
-        next_seq = random.choice(nex_seq_list)
-        tokens = self.lines[next_seq]
-        # print(f"item = {item}, tokens: {tokens}")
-        # print(f"item={item}, next={next_seq}, same_student = {same_student}, {document_id} == {random_document_id}, b. {tokens}")
-        return same_student, tokens
-
-    def truncate_to_max_seq(self, s1, s2):
-        sa = copy.deepcopy(s1)
-        sb = copy.deepcopy(s1)
-        total_allowed_seq = self.seq_len - 3
-        
-        while((len(sa)+len(sb)) > total_allowed_seq):
-            if random.random() < 0.5:
-                sa.pop()
-            else:
-                sb.pop()
-        return sa, sb
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
                 
 class TokenizerDataset(Dataset):
     """
@@ -332,24 +204,15 @@ class TokenizerDataset(Dataset):
         Tokenize the data in the dataset
         
     """
-<<<<<<< HEAD
     def __init__(self, dataset_path, label_path, vocab, seq_len=30):
         self.dataset_path = dataset_path
         self.label_path = label_path
         self.vocab = vocab # Vocab object
         # self.encoder = OneHotEncoder(sparse=False)
-=======
-    def __init__(self, dataset_path, label_path, vocab, seq_len=30, train=True):
-        self.dataset_path = dataset_path
-        self.label_path = label_path
-        self.vocab = vocab # Vocab object
-        self.encoder = OneHotEncoder(sparse_output=False)
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         
         # Related to input dataset file
         self.lines = []
         self.labels = []
-<<<<<<< HEAD
         self.feats = []
         if self.label_path:
             self.label_file = open(self.label_path, "r")
@@ -414,97 +277,21 @@ class TokenizerDataset(Dataset):
             # self.labels = self.encoder.transform(np.array(self.labels).reshape(-1,1))
 
         self.file = open(self.dataset_path, "r")
-=======
-        self.labels = []
-        
-        self.label_file = open(self.label_path, "r")
-        for line in self.label_file:
-            if line:
-                line = line.strip()
-                if not line:
-                    continue
-                self.labels.append(float(line))
-        self.label_file.close()
-        labeler = np.unique(self.labels)
-        self.encoder.fit(labeler.reshape(-1,1))
-        self.labels = self.encoder.transform(np.array(self.labels).reshape(-1,1))
-        # print(f"labels: {self.labels}")
-        
-#         info_file_name = self.dataset_path.split('.')
-#         info_file_name = info_file_name[0]+"_info."+info_file_name[1]
-#         progress = []
-#         with open(info_file_name, "r") as f:
-#             for line in f:
-#                 if line:
-#                     line = line.strip()
-#                     if not line:
-#                         continue
-#                     line = line.split(",")[0]
-#                     pstat = 1 if line == "GRADUATED" else 0
-#                     progress.append(pstat)
-#             f.close()
-            
-#         indices_of_grad = np.where(np.array(progress) == 1)[0]
-#         indices_of_prom = np.where(np.array(progress) == 0)[0]
-        
-#         indices_of_zeros = np.where(np.array(labels) == 0)[0]
-#         indices_of_ones = np.where(np.array(labels) == 1)[0]
-        
-#         number_of_items = min(len(indices_of_zeros), len(indices_of_ones))
-#         # number_of_items = min(len(indices_of_grad), len(indices_of_prom))
-#         print(number_of_items)
-        
-#         indices_of_zeros = indices_of_zeros[:number_of_items]
-#         indices_of_ones = indices_of_ones[:number_of_items]
-#         print(indices_of_zeros)
-#         print(indices_of_ones)
-        
-        # indices_of_grad = indices_of_grad[:number_of_items]
-        # indices_of_prom = indices_of_prom[:number_of_items]
-        # print(indices_of_grad)
-        # print(indices_of_prom)
-
-        self.file = open(self.dataset_path, "r")
-        # index = 0
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         for line in self.file:
             if line:
                 line = line.strip()
                 if line:
                     self.lines.append(line)
-<<<<<<< HEAD
-=======
-                    # if train:
-                    #     if index in indices_of_zeros:
-                    #     # if index in indices_of_prom:
-                    #         self.lines.append(line)
-                    #         self.labels.append(0)
-                    #     if index in indices_of_ones:
-                    #     # if index in indices_of_grad:
-                    #         self.lines.append(line)
-                    #         self.labels.append(1)
-                    # else:
-                    #     self.lines.append(line)
-                    #     self.labels.append(labels[index])
-                        # self.labels.append(progress[index])
-                    # index += 1
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         self.file.close()             
         
         self.len = len(self.lines)
         self.seq_len = seq_len
-<<<<<<< HEAD
         print("Sequence length set at ", self.seq_len, len(self.lines), len(self.labels) if self.label_path else 0)
-=======
-         
-        print("Sequence length set at ", self.seq_len, len(self.lines), len(self.labels))
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         
     def __len__(self):
         return self.len
     
     def __getitem__(self, item):
-<<<<<<< HEAD
         org_line = self.lines[item].split("\t")
         dup_line = []
         opt = False
@@ -527,23 +314,10 @@ class TokenizerDataset(Dataset):
         output = {'input': s1,
                  'label': s1_label,
                   'feat': s1_feat,
-=======
-        
-        s1 = self.vocab.to_seq(self.lines[item], self.seq_len) # This is like tokenizer and adds [CLS] and [SEP].
-        s1_label = self.labels[item]
-        segment_label = [1 for _ in range(len(s1))]
-        
-        padding = [self.vocab.vocab['[PAD]'] for _ in range(self.seq_len - len(s1))]
-        s1.extend(padding), segment_label.extend(padding)
-        
-        output = {'bert_input': s1,
-                 'progress_status': s1_label,
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
                  'segment_label': segment_label}
         return {key: torch.tensor(value) for key, value in output.items()}
         
         
-<<<<<<< HEAD
 class TokenizerDatasetForCalibration(Dataset):
     """
         Class name: TokenizerDataset
@@ -661,9 +435,6 @@ class TokenizerDatasetForCalibration(Dataset):
         
         
         # if __name__ == "__main__":
-=======
-# if __name__ == "__main__":
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
 #     # import pickle
 #     # k = pickle.load(open("dataset/CL4999_1920/unique_steps_list.pkl","rb"))
 #     # print(k)

src/pretrainer.py

@@ -1,6 +1,5 @@
 import torch
 import torch.nn as nn
-<<<<<<< HEAD
 # from torch.nn import functional as F
 from torch.optim import Adam
 from torch.utils.data import DataLoader
@@ -36,75 +35,6 @@ class BERTTrainer:
                  train_dataloader: DataLoader, val_dataloader: DataLoader = None, test_dataloader: DataLoader = None,
                  lr: float = 1e-4, betas=(0.9, 0.999), weight_decay: float = 0.01, warmup_steps=5000,
                  with_cuda: bool = True, cuda_devices=None, log_freq: int = 10, log_folder_path: str = None):
-=======
-from torch.nn import functional as F
-from torch.optim import Adam, SGD
-from torch.utils.data import DataLoader
-import pickle
-
-from bert import BERT
-from seq_model import BERTSM
-from classifier_model import BERTForClassification
-from optim_schedule import ScheduledOptim
-
-import tqdm
-import sys
-
-import numpy as np
-import visualization
-
-from sklearn.metrics import precision_score, recall_score, f1_score
-
-class ECE(nn.Module):
-
-    def __init__(self, n_bins=15):
-        """
-        n_bins (int): number of confidence interval bins
-        """
-        super(ECE, self).__init__()
-        bin_boundaries = torch.linspace(0, 1, n_bins + 1)
-        self.bin_lowers = bin_boundaries[:-1]
-        self.bin_uppers = bin_boundaries[1:]
-
-    def forward(self, logits, labels):
-        softmaxes = F.softmax(logits, dim=1)
-        confidences, predictions = torch.max(softmaxes, 1)
-        labels = torch.argmax(labels,1)
-        accuracies = predictions.eq(labels)
-
-        ece = torch.zeros(1, device=logits.device)
-        for bin_lower, bin_upper in zip(self.bin_lowers, self.bin_uppers):
-            # Calculated |confidence - accuracy| in each bin
-            in_bin = confidences.gt(bin_lower.item()) * confidences.le(bin_upper.item())
-            prop_in_bin = in_bin.float().mean()
-            if prop_in_bin.item() > 0:
-                accuracy_in_bin = accuracies[in_bin].float().mean()
-                avg_confidence_in_bin = confidences[in_bin].mean()
-                ece += torch.abs(avg_confidence_in_bin - accuracy_in_bin) * prop_in_bin
-
-        return ece
-    
-def accurate_nb(preds, labels):
-    pred_flat = np.argmax(preds, axis=1).flatten()
-    labels_flat = np.argmax(labels, axis=1).flatten()
-    labels_flat = labels.flatten()
-    return np.sum(pred_flat == labels_flat)
-
-class BERTTrainer:
-    """
-    # Sequence..
-    
-    BERTTrainer make the pretrained BERT model with two LM training method.
-
-        1. Masked Language Model : 3.3.1 Task #1: Masked LM
-    """
-
-    def __init__(self, bert: BERT, vocab_size: int,
-                 train_dataloader: DataLoader, test_dataloader: DataLoader = None,
-                 lr: float = 1e-4, betas=(0.9, 0.999), weight_decay: float = 0.01, warmup_steps=10000,
-                 with_cuda: bool = True, cuda_devices=None, log_freq: int = 10, same_student_prediction = False,
-                workspace_name=None):
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         """
         :param bert: BERT model which you want to train
         :param vocab_size: total word vocab size
@@ -117,7 +47,6 @@ class BERTTrainer:
         :param log_freq: logging frequency of the batch iteration
         """
 
-<<<<<<< HEAD
         cuda_condition = torch.cuda.is_available() and with_cuda
         self.device = torch.device("cuda:0" if cuda_condition else "cpu")
         print(cuda_condition, " Device used = ", self.device)
@@ -127,33 +56,16 @@ class BERTTrainer:
         # This BERT model will be saved 
         self.bert = bert.to(self.device)
         # Initialize the BERT Sequence Model, with BERT model
-=======
-        # Setup cuda device for BERT training, argument -c, --cuda should be true
-        cuda_condition = torch.cuda.is_available() and with_cuda
-        self.device = torch.device("cuda:0" if cuda_condition else "cpu")
-        print("Device used = ", self.device)
-
-        # This BERT model will be saved every epoch
-        self.bert = bert
-        # Initialize the BERT Language Model, with BERT model
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         self.model = BERTSM(bert, vocab_size).to(self.device)
 
         # Distributed GPU training if CUDA can detect more than 1 GPU
         if with_cuda and torch.cuda.device_count() > 1:
             print("Using %d GPUS for BERT" % torch.cuda.device_count())
-<<<<<<< HEAD
             self.model = nn.DataParallel(self.model, device_ids=available_gpus)
 
         # Setting the train, validation and test data loader
         self.train_data = train_dataloader
         self.val_data = val_dataloader
-=======
-            self.model = nn.DataParallel(self.model, device_ids=cuda_devices)
-
-        # Setting the train and test data loader
-        self.train_data = train_dataloader
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         self.test_data = test_dataloader
 
         # Setting the Adam optimizer with hyper-param
@@ -164,7 +76,6 @@ class BERTTrainer:
         self.criterion = nn.NLLLoss(ignore_index=0)
     
         self.log_freq = log_freq
-<<<<<<< HEAD
         self.log_folder_path = log_folder_path
         # self.workspace_name = workspace_name
         self.save_model = False
@@ -175,18 +86,11 @@ class BERTTrainer:
             f.close()
         self.start_time = time.time()
 
-=======
-        self.same_student_prediction = same_student_prediction
-        self.workspace_name = workspace_name
-        self.save_model = False
-        self.avg_loss = 10000
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         print("Total Parameters:", sum([p.nelement() for p in self.model.parameters()]))
 
     def train(self, epoch):
         self.iteration(epoch, self.train_data)
 
-<<<<<<< HEAD
     def val(self, epoch):
         if epoch == 0:
             self.avg_loss = 10000
@@ -196,12 +100,6 @@ class BERTTrainer:
         self.iteration(epoch, self.test_data, phase="test")
 
     def iteration(self, epoch, data_loader, phase="train"):
-=======
-    def test(self, epoch):
-        self.iteration(epoch, self.test_data, train=False)
-
-    def iteration(self, epoch, data_loader, train=True):
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         """
         loop over the data_loader for training or testing
         if on train status, backward operation is activated
@@ -212,7 +110,6 @@ class BERTTrainer:
         :param train: boolean value of is train or test
         :return: None
         """
-<<<<<<< HEAD
         
         # self.log_file = f"{self.workspace_name}/logs/{self.code}/log_{phase}_pretrained.txt"
         # bert_hidden_representations = [] can be used
@@ -235,39 +132,10 @@ class BERTTrainer:
         else:
             self.model.eval()
         with open(self.log_folder_path+f"/log_{phase}_pretrained.txt", 'a') as f:
-=======
-        str_code = "train" if train else "test"
-        code = "masked_prediction" if self.same_student_prediction else "masked"
-       
-        self.log_file = f"{self.workspace_name}/logs/{code}/log_{str_code}_pretrained.txt"
-        bert_hidden_representations = []
-        if epoch == 0:
-            f = open(self.log_file, 'w')
-            f.close()
-            if not train:
-                self.avg_loss = 10000
-        # Setting the tqdm progress bar
-        data_iter = tqdm.tqdm(enumerate(data_loader),
-                              desc="EP_%s:%d" % (str_code, epoch),
-                              total=len(data_loader),
-                              bar_format="{l_bar}{r_bar}")
-
-        avg_loss_mask = 0.0
-        total_correct_mask = 0
-        total_element_mask = 0
-        
-        avg_loss_pred = 0.0
-        total_correct_pred = 0
-        total_element_pred = 0
-        
-        avg_loss = 0.0
-        with open(self.log_file, 'a') as f:
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
             sys.stdout = f
             for i, data in data_iter:
                 # 0. batch_data will be sent into the device(GPU or cpu)
                 data = {key: value.to(self.device) for key, value in data.items()}
-<<<<<<< HEAD
                 
                 # 1. forward masked_sm model
                 # mask_sm_output is log-probabilities output
@@ -280,38 +148,10 @@ class BERTTrainer:
                     
                 # 3. backward and optimization only in train
                 if phase == "train":
-=======
-
-                # 1. forward the next_sentence_prediction and masked_lm model
-                # next_sent_output, mask_lm_output = self.model.forward(data["bert_input"], data["segment_label"])
-                if self.same_student_prediction:
-                    bert_hidden_rep, mask_lm_output, same_student_output = self.model.forward(data["bert_input"], data["segment_label"], self.same_student_prediction)
-                else:
-                    bert_hidden_rep, mask_lm_output = self.model.forward(data["bert_input"], data["segment_label"], self.same_student_prediction)
-
-                embeddings = [h for h in bert_hidden_rep.cpu().detach().numpy()]
-                bert_hidden_representations.extend(embeddings)
-
-
-                # 2-2. NLLLoss of predicting masked token word
-                mask_loss = self.criterion(mask_lm_output.transpose(1, 2), data["bert_label"])
-
-                # 2-3. Adding next_loss and mask_loss : 3.4 Pre-training Procedure
-                if self.same_student_prediction:
-                    # 2-1. NLL(negative log likelihood) loss of is_next classification result
-                    same_student_loss = self.criterion(same_student_output, data["is_same_student"])
-                    loss = same_student_loss + mask_loss
-                else:
-                    loss = mask_loss
-
-                # 3. backward and optimization only in train
-                if train:
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
                     self.optim_schedule.zero_grad()
                     loss.backward()
                     self.optim_schedule.step_and_update_lr()
 
-<<<<<<< HEAD
                 # tokens with highest log-probabilities creates a predicted sequence
                 pred_tokens = torch.argmax(mask_sm_output, dim=-1) 
                 mask_correct = (data["bert_label"] == pred_tokens) & data["masked_pos"]
@@ -348,69 +188,6 @@ class BERTTrainer:
             if self.avg_loss > (avg_loss / len(data_iter)):
                 self.save_model = True
                 self.avg_loss = (avg_loss / len(data_iter))
-=======
-
-                non_zero_mask = (data["bert_label"] != 0).float()
-                predictions = torch.argmax(mask_lm_output, dim=-1)
-                predicted_masked = predictions*non_zero_mask
-                mask_correct = ((data["bert_label"] == predicted_masked)*non_zero_mask).sum().item()
-
-                avg_loss_mask += loss.item()
-                total_correct_mask += mask_correct
-                total_element_mask += non_zero_mask.sum().item()
-
-                post_fix = {
-                    "epoch": epoch,
-                    "iter": i,
-                    "avg_loss": avg_loss_mask / (i + 1),
-                    "avg_acc_mask": total_correct_mask / total_element_mask * 100,
-                    "loss": loss.item()
-                }
-
-                # next sentence prediction accuracy
-                if self.same_student_prediction:
-                    correct = same_student_output.argmax(dim=-1).eq(data["is_same_student"]).sum().item()
-                    avg_loss_pred += loss.item()
-                    total_correct_pred += correct
-                    total_element_pred += data["is_same_student"].nelement()
-                # correct = next_sent_output.argmax(dim=-1).eq(data["is_next"]).sum().item()
-                    post_fix["avg_loss"] = avg_loss_pred / (i + 1)
-                    post_fix["avg_acc_pred"] = total_correct_pred / total_element_pred * 100
-                    post_fix["loss"] = loss.item()
-
-                avg_loss +=loss.item()
-
-                if i % self.log_freq == 0:
-                    data_iter.write(str(post_fix))
-                # if not train and epoch > 20 :
-                #     pickle.dump(mask_lm_output.cpu().detach().numpy(), open(f"logs/mask/mask_out_e{epoch}_{i}.pkl","wb"))
-                #     pickle.dump(data["bert_label"].cpu().detach().numpy(), open(f"logs/mask/label_e{epoch}_{i}.pkl","wb"))
-            
-            final_msg = {
-                "epoch": f"EP{epoch}_{str_code}",
-                "avg_loss": avg_loss / len(data_iter),
-                "total_masked_acc": total_correct_mask * 100.0 / total_element_mask
-            }
-            if self.same_student_prediction:
-                final_msg["total_prediction_acc"] = total_correct_pred * 100.0 / total_element_pred
-
-            print(final_msg)
-            # print("EP%d_%s, avg_loss=" % (epoch, str_code), avg_loss / len(data_iter), "total_masked_acc=", total_correct_mask * 100.0 / total_element_mask, "total_prediction_acc=", total_correct_pred * 100.0 / total_element_pred)
-        # else:
-        #     print("EP%d_%s, avg_loss=" % (epoch, str_code), avg_loss / len(data_iter), "total_masked_acc=", total_correct_mask * 100.0 / total_element_mask)
-        # print("EP%d_%s, " % (epoch, str_code))
-        
-            f.close()
-        sys.stdout = sys.__stdout__
-        self.save_model = False
-        if self.avg_loss > (avg_loss / len(data_iter)):
-            self.save_model = True
-            self.avg_loss = (avg_loss / len(data_iter))
-
-        # pickle.dump(bert_hidden_representations, open(f"embeddings/{code}/{str_code}_embeddings_{epoch}.pkl","wb"))
-
-        
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
 
     def save(self, epoch, file_path="output/bert_trained.model"):
         """
@@ -432,12 +209,8 @@ class BERTFineTuneTrainer:
     def __init__(self, bert: BERT, vocab_size: int,
                  train_dataloader: DataLoader, test_dataloader: DataLoader = None,
                  lr: float = 1e-4, betas=(0.9, 0.999), weight_decay: float = 0.01, warmup_steps=10000,
-<<<<<<< HEAD
                  with_cuda: bool = True, cuda_devices=None, log_freq: int = 10, workspace_name=None, 
                  num_labels=2, log_folder_path: str = None):
-=======
-                 with_cuda: bool = True, cuda_devices=None, log_freq: int = 10, workspace_name=None, num_labels=2):
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         """
         :param bert: BERT model which you want to train
         :param vocab_size: total word vocab size
@@ -453,7 +226,6 @@ class BERTFineTuneTrainer:
         # Setup cuda device for BERT training, argument -c, --cuda should be true
         cuda_condition = torch.cuda.is_available() and with_cuda
         self.device = torch.device("cuda:0" if cuda_condition else "cpu")
-<<<<<<< HEAD
         print(cuda_condition, " Device used = ", self.device)
         
         available_gpus = list(range(torch.cuda.device_count()))
@@ -462,6 +234,16 @@ class BERTFineTuneTrainer:
         self.bert = bert
         for param in self.bert.parameters():
             param.requires_grad = False
+            
+        # for name, param in self.bert.named_parameters():
+        #     if '.attention.linear_layers.0' in name or \
+        #        '.attention.linear_layers.1' in name or \
+        #        '.attention.linear_layers.2' in name:
+        #     # if 'transformer_blocks.' in name:# or \
+        #        # 'transformer_blocks.3.' in name:
+        #     # if '2.attention.linear_layers.' in name or \
+        #        # '3.attention.linear_layers.' in name:
+        #         param.requires_grad = True
         # Initialize the BERT Language Model, with BERT model
         # self.model = BERTForClassification(self.bert, vocab_size, num_labels).to(self.device)
         # self.model = BERTForClassificationWithFeats(self.bert, num_labels, 8).to(self.device)
@@ -748,48 +530,11 @@ class BERTFineTuneTrainer1:
         for fi in ['train', 'test']: #'val', 
             f = open(self.log_folder_path+f"/log_{fi}_finetuned.txt", 'w')
             f.close()
-=======
-        print("Device used = ", self.device)
-
-        # This BERT model will be saved every epoch
-        self.bert = bert
-        # for param in self.bert.parameters():
-        #     param.requires_grad = False
-        # Initialize the BERT Language Model, with BERT model
-        self.model = BERTForClassification(self.bert, vocab_size, num_labels).to(self.device)
-
-        # Distributed GPU training if CUDA can detect more than 1 GPU
-        if with_cuda and torch.cuda.device_count() > 1:
-            print("Using %d GPUS for BERT" % torch.cuda.device_count())
-            self.model = nn.DataParallel(self.model, device_ids=cuda_devices)
-
-        # Setting the train and test data loader
-        self.train_data = train_dataloader
-        self.test_data = test_dataloader
-    
-        self.optim = Adam(self.model.parameters(), lr=lr, weight_decay=weight_decay, eps=1e-9)
-        # self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=2, factor=0.1)
-        
-        if num_labels == 1:
-            self.criterion = nn.MSELoss()
-        elif num_labels == 2:
-            self.criterion = nn.CrossEntropyLoss()
-        elif num_labels > 2:
-            self.criterion = nn.BCEWithLogitsLoss()
-        
-        self.ece_criterion = ECE().to(self.device)
-        
-        self.log_freq = log_freq
-        self.workspace_name = workspace_name
-        self.save_model = False
-        self.avg_loss = 10000
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         print("Total Parameters:", sum([p.nelement() for p in self.model.parameters()]))
 
     def train(self, epoch):
         self.iteration(epoch, self.train_data)
 
-<<<<<<< HEAD
     # def val(self, epoch):
     #     self.iteration(epoch, self.val_data, phase="val")
         
@@ -799,12 +544,6 @@ class BERTFineTuneTrainer1:
         self.iteration(epoch, self.test_data, phase="test")
 
     def iteration(self, epoch, data_loader, phase="train"):
-=======
-    def test(self, epoch):
-        self.iteration(epoch, self.test_data, train=False)
-
-    def iteration(self, epoch, data_loader, train=True):
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         """
         loop over the data_loader for training or testing
         if on train status, backward operation is activated
@@ -815,26 +554,10 @@ class BERTFineTuneTrainer1:
         :param train: boolean value of is train or test
         :return: None
         """
-<<<<<<< HEAD
         
         # Setting the tqdm progress bar
         data_iter = tqdm.tqdm(enumerate(data_loader),
                               desc="EP_%s:%d" % (phase, epoch),
-=======
-        str_code = "train" if train else "test"
-        
-        self.log_file = f"{self.workspace_name}/logs/masked/log_{str_code}_FS_finetuned.txt"
-
-        if epoch == 0:
-            f = open(self.log_file, 'w')
-            f.close()
-            if not train:
-                self.avg_loss = 10000
-            
-        # Setting the tqdm progress bar
-        data_iter = tqdm.tqdm(enumerate(data_loader),
-                              desc="EP_%s:%d" % (str_code, epoch),
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
                               total=len(data_loader),
                               bar_format="{l_bar}{r_bar}")
 
@@ -843,7 +566,6 @@ class BERTFineTuneTrainer1:
         total_element = 0
         plabels = []
         tlabels = []
-<<<<<<< HEAD
         probabs = []
 
         if phase == "train":
@@ -864,43 +586,10 @@ class BERTFineTuneTrainer1:
                         logits = self.model.forward(data["input"], data["segment_label"])#, data["feat"])
 
                 loss = self.criterion(logits, data["label"])
-=======
-        eval_accurate_nb = 0
-        nb_eval_examples = 0
-        logits_list = []
-        labels_list = []
-        
-        if train:
-            self.model.train()
-        else:
-            self.model.eval()
-            
-        with open(self.log_file, 'a') as f:
-            sys.stdout = f
-            
-            for i, data in data_iter:
-                # 0. batch_data will be sent into the device(GPU or cpu)
-                data = {key: value.to(self.device) for key, value in data.items()}
-                if train:
-                    h_rep, logits = self.model.forward(data["bert_input"], data["segment_label"])
-                else:
-                    with torch.no_grad():
-                        h_rep, logits = self.model.forward(data["bert_input"], data["segment_label"])
-                    # print(logits, logits.shape)
-                    logits_list.append(logits.cpu())
-                    labels_list.append(data["progress_status"].cpu())
-                # print(">>>>>>>>>>>>", progress_output)
-                # print(f"{epoch}---nelement--- {data['progress_status'].nelement()}")
-                # print(data["progress_status"].shape, logits.shape)
-                progress_loss = self.criterion(logits, data["progress_status"])
-                loss = progress_loss
-                
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
                 if torch.cuda.device_count() > 1:
                     loss = loss.mean()
 
                 # 3. backward and optimization only in train
-<<<<<<< HEAD
                 if phase == "train":
                     self.optim_schedule.zero_grad()
                     loss.backward()
@@ -969,108 +658,10 @@ class BERTFineTuneTrainer1:
         sys.stdout = sys.__stdout__
         
         if phase == "test":
-=======
-                if train:
-                    self.optim.zero_grad()
-                    loss.backward()
-                    torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)
-                    self.optim.step()
-
-                # progress prediction accuracy
-                # correct = progress_output.argmax(dim=-1).eq(data["progress_status"]).sum().item()
-                probs = nn.LogSoftmax(dim=-1)(logits)
-                predicted_labels = torch.argmax(probs, dim=-1)
-                true_labels = torch.argmax(data["progress_status"], dim=-1)
-                plabels.extend(predicted_labels.cpu().numpy())
-                tlabels.extend(true_labels.cpu().numpy())
-
-                # print(">>>>>>>>>>>>>>", predicted_labels, true_labels)
-                # Compare predicted labels to true labels and calculate accuracy
-                correct = (predicted_labels == true_labels).sum().item()
-                avg_loss += loss.item()
-                total_correct += correct
-                total_element += true_labels.nelement()
-
-                if train:
-                    post_fix = {
-                        "epoch": epoch,
-                        "iter": i,
-                        "avg_loss": avg_loss / (i + 1),
-                        "avg_acc": total_correct / total_element * 100,
-                        "loss": loss.item()
-                    }
-                else:
-                    logits = logits.detach().cpu().numpy()
-                    label_ids = data["progress_status"].to('cpu').numpy()
-                    tmp_eval_nb = accurate_nb(logits, label_ids)
-
-                    eval_accurate_nb += tmp_eval_nb
-                    nb_eval_examples += label_ids.shape[0]
-                    
-                    total_element += data["progress_status"].nelement()
-                    # avg_loss += loss.item()
-                    
-                    post_fix = {
-                        "epoch": epoch,
-                        "iter": i,
-                        "avg_loss": avg_loss / (i + 1),
-                        "avg_acc": tmp_eval_nb / total_element * 100,
-                        "loss": loss.item()
-                    }
-
-                
-                if i % self.log_freq == 0:
-                    data_iter.write(str(post_fix))
-            
-            # precisions = precision_score(plabels, tlabels, average="weighted")
-            # recalls = recall_score(plabels, tlabels, average="weighted")
-            f1_scores = f1_score(plabels, tlabels, average="weighted")
-            if train:
-                final_msg = {
-                "epoch": f"EP{epoch}_{str_code}",
-                "avg_loss": avg_loss / len(data_iter),
-                "total_acc": total_correct * 100.0 / total_element,
-                # "precisions": precisions,
-                # "recalls": recalls,
-                "f1_scores": f1_scores
-                }
-            else:
-                eval_accuracy = eval_accurate_nb/nb_eval_examples
-                
-                logits_ece = torch.cat(logits_list)
-                labels_ece = torch.cat(labels_list)
-                ece = self.ece_criterion(logits_ece, labels_ece).item()
-                final_msg = {
-                    "epoch": f"EP{epoch}_{str_code}",
-                    "eval_accuracy": eval_accuracy,
-                    "ece": ece,
-                    "avg_loss": avg_loss / len(data_iter),
-                    # "precisions": precisions,
-                    # "recalls": recalls,
-                    "f1_scores": f1_scores
-                }
-                if self.save_model:
-                    conf_hist = visualization.ConfidenceHistogram()
-                    plt_test = conf_hist.plot(np.array(logits_ece), np.array(labels_ece), title= f"Confidence Histogram {epoch}")
-                    plt_test.savefig(f"{self.workspace_name}/plots/confidence_histogram/FS/conf_histogram_test_{epoch}.png",bbox_inches='tight')
-                    plt_test.close()
-
-                    rel_diagram = visualization.ReliabilityDiagram()
-                    plt_test_2 = rel_diagram.plot(np.array(logits_ece), np.array(labels_ece),title=f"Reliability Diagram {epoch}")
-                    plt_test_2.savefig(f"{self.workspace_name}/plots/confidence_histogram/FS/rel_diagram_test_{epoch}.png",bbox_inches='tight')
-                    plt_test_2.close()
-            print(final_msg)
-
-            # print("EP%d_%s, avg_loss=" % (epoch, str_code), avg_loss / len(data_iter), "total_acc=", total_correct * 100.0 / total_element)
-            f.close()
-        sys.stdout = sys.__stdout__
-        if train:
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
             self.save_model = False
             if self.avg_loss > (avg_loss / len(data_iter)):
                 self.save_model = True
                 self.avg_loss = (avg_loss / len(data_iter))
-<<<<<<< HEAD
             
     def iteration_1(self, epoch_idx, data):
         try:
@@ -1094,11 +685,6 @@ class BERTFineTuneTrainer1:
             print(f"Error during iteration: {e}")
             raise
 
-=======
-        
-                # plt_test.show()
-        # print("EP%d_%s, " % (epoch, str_code))
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
 
     def save(self, epoch, file_path="output/bert_fine_tuned_trained.model"):
         """
@@ -1113,7 +699,6 @@ class BERTFineTuneTrainer1:
         self.model.to(self.device)
         print("EP:%d Model Saved on:" % epoch, output_path)
         return output_path        
-<<<<<<< HEAD
     
 
 class BERTAttention:
@@ -1221,5 +806,3 @@ class BERTAttention:
         
         
         
-=======
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896

src/seq_model.py

@@ -1,10 +1,6 @@
 import torch.nn as nn
 
-<<<<<<< HEAD
 from .bert import BERT
-=======
-from bert import BERT
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
 
 
 class BERTSM(nn.Module):
@@ -22,23 +18,10 @@ class BERTSM(nn.Module):
         super().__init__()
         self.bert = bert
         self.mask_lm = MaskedSequenceModel(self.bert.hidden, vocab_size)
-<<<<<<< HEAD
         
     def forward(self, x, segment_label):
         x = self.bert(x, segment_label)
         return self.mask_lm(x), x[:, 0]
-=======
-        self.same_student = SameStudentPrediction(self.bert.hidden)
-        
-    def forward(self, x, segment_label, pred=False):
-        x = self.bert(x, segment_label)
-        # torch.Size([32, 200, 512])
-        # print("????????????   ",x.shape)
-        if pred:
-            return x[:, 0], self.mask_lm(x), self.same_student(x)
-        else:
-            return x[:, 0], self.mask_lm(x)
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
 
     
 class MaskedSequenceModel(nn.Module):
@@ -57,23 +40,4 @@ class MaskedSequenceModel(nn.Module):
         self.softmax = nn.LogSoftmax(dim=-1)
 
     def forward(self, x):
-<<<<<<< HEAD
-        return self.softmax(self.linear(x))
-=======
-        return self.softmax(self.linear(x))
-
-    
-class SameStudentPrediction(nn.Module):
-
-    def __init__(self, hidden):
-        """
-        :param hidden: BERT model output size
-        """
-        super().__init__()
-        self.linear = nn.Linear(hidden, 2)
-        self.softmax = nn.LogSoftmax(dim=-1)
-
-    def forward(self, x):
-        return self.softmax(self.linear(x[:, 0]))
-
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
+        return self.softmax(self.linear(x))

src/transformer.py

@@ -1,12 +1,7 @@
 import torch.nn as nn
 
-<<<<<<< HEAD
 from .attention import MultiHeadedAttention
 from .transformer_component import SublayerConnection, PositionwiseFeedForward
-=======
-from attention import MultiHeadedAttention
-from transformer_component import SublayerConnection, PositionwiseFeedForward
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
 
 class TransformerBlock(nn.Module):
     """
@@ -30,12 +25,8 @@ class TransformerBlock(nn.Module):
         self.dropout = nn.Dropout(p=dropout)
 
     def forward(self, x, mask):
-<<<<<<< HEAD
         attn_output, p_attn = self.attention.forward(x, x, x, mask=mask)
         self.p_attn = p_attn.cpu().detach().numpy()
         x = self.input_sublayer(x, lambda _x: attn_output)
-=======
-        x = self.input_sublayer(x, lambda _x: self.attention.forward(_x, _x, _x, mask=mask))
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
         x = self.output_sublayer(x, self.feed_forward)
         return self.dropout(x)

src/vocab.py

@@ -1,22 +1,16 @@
 import collections
 import tqdm
-<<<<<<< HEAD
 import os
 from pathlib import Path
 
 head_directory = Path(__file__).resolve().parent.parent
 # print(head_directory)
 os.chdir(head_directory)
-=======
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
 
 class Vocab(object):
     """
     Special tokens predefined in the vocab file are:
-<<<<<<< HEAD
     -[PAD]
-=======
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
     -[UNK]
     -[MASK]
     -[CLS]
@@ -48,11 +42,7 @@ class Vocab(object):
         words = [self.invocab[index] if index < len(self.invocab) 
                  else "[%d]" % index for index in seq ]
         
-<<<<<<< HEAD
         return words #" ".join(words)
-=======
-        return " ".join(words)
->>>>>>> bffd3381ccb717f802fe651d4111ec0a268e3896
     
 
 # if __init__ == "__main__":