Update PrateritumGPT.py

PrateritumGPT.py

@@ -35,13 +35,13 @@ labels = []
 for i in reader:
     k = []
     for j in i[2]:
-        k += [tokens.index(j)]
-    k += [len(tokens) + 1] * (25 - len(k))
+        k += [tokens.index(j)+1]
+    k += [0] * (25 - len(k))
     features += [torch.Tensor(k)]
     k = []
     for j in i[8]:
-        k += [tokens.index(j)]
-    k += [len(tokens) + 1] * (25 - len(k))
+        k += [tokens.index(j)+1]
+    k += [0] * (25 - len(k))
     labels += [torch.Tensor(k)]
 
 MyDataset = CSVDataset(features=features, labels=labels)
@@ -51,21 +51,26 @@ class TransformerModel(nn.Module):
         super().__init__()
         self.custom_embedding = nn.Embedding(vocab_size, emb_dim)
         self.pos_encoder = PositionalEncoding(emb_dim, dropout)
-        encoder_layer = nn.TransformerEncoderLayer(emb_dim, nhead, dim_feedforward, dropout)
+        encoder_layer = nn.TransformerEncoderLayer(emb_dim, nhead, dim_feedforward, dropout, batch_first=True)
         self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_encoder_layers)
-        decoder_layer = nn.TransformerDecoderLayer(emb_dim, nhead, dim_feedforward, dropout)
+        decoder_layer = nn.TransformerDecoderLayer(emb_dim, nhead, dim_feedforward, dropout, batch_first=True)
         self.transformer_decoder = nn.TransformerDecoder(decoder_layer, num_decoder_layers)
         self.output_layer = nn.Linear(emb_dim, vocab_size)
 
     def forward(self, src, tgt, src_mask=None, tgt_mask=None, memory_mask=None, src_key_padding_mask=None, tgt_key_padding_mask=None, memory_key_padding_mask=None):
-        src_emb = self.pos_encoder(self.custom_embedding(src.long()))
-        tgt_emb = self.pos_encoder(self.custom_embedding(tgt.long()))
+        src_emb = self.custom_embedding(src.long())
+        print("Source Embedding:", src_emb.shape)
+        src_emb = self.pos_encoder(src_emb)
+        print("Source Embedding:", src_emb.shape)
+        tgt_emb = self.custom_embedding(tgt.long())
+        print("Target Embedding:", tgt_emb.shape)
+        tgt_emb = self.pos_encoder(tgt_emb)
+        print("Target Embedding:", tgt_emb.shape)
         encoder_output = self.transformer_encoder(src_emb, src_mask, src_key_padding_mask)
         decoder_output = self.transformer_decoder(tgt_emb, encoder_output, tgt_mask, memory_mask, tgt_key_padding_mask, memory_key_padding_mask)
         output = self.output_layer(decoder_output)
         return output
 
-# Définition de la classe PositionalEncoding (identique à l'exemple précédent)
 class PositionalEncoding(nn.Module):
     def __init__(self, d_model, dropout=0.1, max_len=5000):
         super(PositionalEncoding, self).__init__()
@@ -76,14 +81,13 @@ class PositionalEncoding(nn.Module):
         div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
         pe[:, 0::2] = torch.sin(position * div_term)
         pe[:, 1::2] = torch.cos(position * div_term)
-        pe = pe.unsqueeze(0).transpose(0, 1)
+        pe = pe.unsqueeze(0)
         self.register_buffer('pe', pe)
 
     def forward(self, x):
-        x = x + self.pe[:x.size(0), :]
+        x = x + self.pe[:, :x.size(1), :]
         return self.dropout(x)
-
-# Préparation des données
+    
 def collate_fn(batch):
     inputs = [item[0] for item in batch]
     targets = [item[1] for item in batch]
@@ -93,8 +97,7 @@ def collate_fn(batch):
 
 train_loader = DataLoader(MyDataset, batch_size=32, shuffle=True, collate_fn=collate_fn)
 
-# Définition du modèle, de la fonction de perte et de l'optimiseur
-model = TransformerModel(vocab_size=len(tokens) + 2, emb_dim=512, nhead=8, num_encoder_layers=6, num_decoder_layers=6, dim_feedforward=2048)
+model = TransformerModel(vocab_size=len(tokens)+1, emb_dim=32, nhead=8, num_encoder_layers=6, num_decoder_layers=6, dim_feedforward=2048)
 loss_fn = nn.CrossEntropyLoss()
 optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
 
@@ -105,18 +108,21 @@ for epoch in range(epochs):
 
     for batch_idx, (inputs, targets) in enumerate(train_loader):
 
-        print(inputs.shape,targets.shape)
+        for i in range(1,targets.shape[1]):
 
-        optimizer.zero_grad()
-        output = model(inputs, targets[:, :-1])  # Shifted targets
-        output = output.transpose(1, 2)  # Adjust shape for loss function
-        loss = loss_fn(output, targets[:, 1:].long())  # Shifted targets
-        loss.backward()
-        optimizer.step()
+            optimizer.zero_grad()
+            output = model(inputs, targets[:, :i])  # Shifted targets
+            output = output.transpose(1, 2)  # Adjust shape for loss function
+            #loss = loss_fn(output, targets[:, i].unsqueeze(1).long())  # Shifted targets
+            print(output.shape)
+            print(targets[:, i].unsqueeze(1).long().shape)
+            loss = loss_fn(output, targets[:, i].unsqueeze(1).long())
+            loss.backward()
+            optimizer.step()
 
-        total_loss += loss.item()
+            total_loss += loss.item()
 
-        if batch_idx % 100 == 0:
-            print(f"Epoch {epoch + 1}/{epochs}, Batch {batch_idx}/{len(train_loader)}, Loss: {total_loss / (batch_idx + 1)}")
+            if batch_idx % 100 == 0:
+                print(f"Epoch {epoch + 1}/{epochs}, Batch {batch_idx}/{len(train_loader)}, Loss: {total_loss / (batch_idx + 1)}")
 
     print(f"Epoch {epoch + 1}/{epochs}, Loss: {total_loss / len(train_loader)}")