Upload main_.py with huggingface_hub

main_.py

@@ -110,8 +110,11 @@ if __name__ == '__main__':
     print('------------------------------------')
     print(type(eval_data))
     print(type(eval_data[0]))
-
+    print(len(eval_data[0]))
     print(len(eval_data))
+    block = 2000
+    for i in range(len(eval_data)):
+        eval_data[i] = eval_data[i][:block]
     eval_set = AITWDatasetImg(
         eval_data,
         tokenizer,
@@ -119,17 +122,15 @@ if __name__ == '__main__':
         args.output_len
     )
     test_data = load_data(args, "test")
+    for i in range(len(test_data)):
+        test_data[i] = test_data[i][:block]
     test_set = AITWDatasetImg(
         test_data,
         tokenizer,
         args.input_len,
         args.output_len
     )
-    block = 2000
-    for i in range(len(test_set)):
-        test_set[i] = test_set[i][:block]
-    for i in range(len(eval_set)):
-        eval_set[i] = eval_set[i][:block]
+    
     datacollator = DataCollatorForSeq2Seq(tokenizer)
     print("model parameters: ", model.num_parameters())