Training in progress, step 500

BluebrainAI___wikitext-103-raw-v1-seq1024-tokenized-grouped/default/0.0.0/e6ef704dbf95ade0b1fde03c931b3112df6ad731/dataset_info.json

@@ -0,0 +1 @@
+{"description": "", "citation": "", "homepage": "", "license": "", "features": {"input_ids": {"feature": {"dtype": "int32", "_type": "Value"}, "_type": "Sequence"}, "attention_mask": {"feature": {"dtype": "int8", "_type": "Value"}, "_type": "Sequence"}}, "builder_name": "parquet", "dataset_name": "wikitext-103-raw-v1-seq1024-tokenized-grouped", "config_name": "default", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"validation": {"name": "validation", "num_bytes": 5997096, "num_examples": 1141, "dataset_name": "wikitext-103-raw-v1-seq1024-tokenized-grouped"}, "train": {"name": "train", "num_bytes": 599152464, "num_examples": 113994, "shard_lengths": [96000, 17994], "dataset_name": "wikitext-103-raw-v1-seq1024-tokenized-grouped"}}, "download_checksums": {"hf://datasets/BluebrainAI/wikitext-103-raw-v1-seq1024-tokenized-grouped@e6ef704dbf95ade0b1fde03c931b3112df6ad731/data/validation-00000-of-00001.parquet": {"num_bytes": 2318981, "checksum": null}, "hf://datasets/BluebrainAI/wikitext-103-raw-v1-seq1024-tokenized-grouped@e6ef704dbf95ade0b1fde03c931b3112df6ad731/data/train-00000-of-00001.parquet": {"num_bytes": 231535449, "checksum": null}}, "download_size": 233854430, "dataset_size": 605149560, "size_in_bytes": 839003990}

BluebrainAI___wikitext-103-raw-v1-seq1024-tokenized-grouped/default/0.0.0/e6ef704dbf95ade0b1fde03c931b3112df6ad731/wikitext-103-raw-v1-seq1024-tokenized-grouped-train-00000-of-00002.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31fa9caad52ee8eb6cc36d08cfe1ee76cdfe3366db187b8f87fc84f8f59f9106
+size 492317736

BluebrainAI___wikitext-103-raw-v1-seq1024-tokenized-grouped/default/0.0.0/e6ef704dbf95ade0b1fde03c931b3112df6ad731/wikitext-103-raw-v1-seq1024-tokenized-grouped-train-00001-of-00002.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a8e3a8d0170e5706b6aae210f6762884c873c66754a7eeab5fd4379cabf4286
+size 92279256

BluebrainAI___wikitext-103-raw-v1-seq1024-tokenized-grouped/default/0.0.0/e6ef704dbf95ade0b1fde03c931b3112df6ad731/wikitext-103-raw-v1-seq1024-tokenized-grouped-validation.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61e41d26a8c63613e66724bf11ada0918b7180bedbe2b5f6a9cc7fe9dc003352
+size 5852200

all_results.json

@@ -1,17 +1,17 @@
 {
     "epoch": 5.0,
-    "eval_accuracy": 0.41983802858530744,
-    "eval_bleu": 0.14543247859716402,
-    "eval_loss": 3.180941343307495,
-    "eval_perplexity": 24.06940046017307,
-    "eval_runtime": 11.0676,
+    "eval_accuracy": 0.41932913712054815,
+    "eval_bleu": 0.1439933070551526,
+    "eval_loss": 3.1860642433166504,
+    "eval_perplexity": 24.19302197197161,
+    "eval_runtime": 11.0615,
     "eval_samples": 1141,
-    "eval_samples_per_second": 103.094,
-    "eval_steps_per_second": 1.626,
-    "perplexity": 24.06940046017307,
+    "eval_samples_per_second": 103.151,
+    "eval_steps_per_second": 1.627,
+    "perplexity": 24.19302197197161,
     "total_flos": 1.0586630697202483e+18,
-    "train_loss": 1.7646983184129434,
-    "train_runtime": 7367.504,
-    "train_samples_per_second": 77.363,
-    "train_steps_per_second": 1.209
+    "train_loss": 3.7704881070840237,
+    "train_runtime": 13409.8958,
+    "train_samples_per_second": 42.504,
+    "train_steps_per_second": 0.664
 }

eval_metrics.json

@@ -0,0 +1,10 @@
+{
+    "eval_loss": 9.138617515563965,
+    "eval_model_preparation_time": 0.0028,
+    "eval_accuracy": 0.003479138448463602,
+    "eval_perplexity": 9307.888222218875,
+    "eval_bleu": 0.0006759862222758408,
+    "eval_runtime": 6.6151,
+    "eval_samples_per_second": 172.483,
+    "eval_steps_per_second": 2.721
+}

eval_results.json

@@ -1,12 +1,12 @@
 {
     "epoch": 5.0,
-    "eval_accuracy": 0.41983802858530744,
-    "eval_bleu": 0.14543247859716402,
-    "eval_loss": 3.180941343307495,
-    "eval_perplexity": 24.06940046017307,
-    "eval_runtime": 11.0676,
+    "eval_accuracy": 0.41932913712054815,
+    "eval_bleu": 0.1439933070551526,
+    "eval_loss": 3.1860642433166504,
+    "eval_perplexity": 24.19302197197161,
+    "eval_runtime": 11.0615,
     "eval_samples": 1141,
-    "eval_samples_per_second": 103.094,
-    "eval_steps_per_second": 1.626,
-    "perplexity": 24.06940046017307
+    "eval_samples_per_second": 103.151,
+    "eval_steps_per_second": 1.627,
+    "perplexity": 24.19302197197161
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9062c5cf6e74adbf118174e57a33cbd4d8758e18adb16a98323b4894d9c43298
+oid sha256:8022a0380584f3db97c80c2aa17051382e9ba2c24c5e39b4212b39d88a89bdd2
 size 1419322880

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 5.0,
     "total_flos": 1.0586630697202483e+18,
-    "train_loss": 1.7646983184129434,
-    "train_runtime": 7367.504,
-    "train_samples_per_second": 77.363,
-    "train_steps_per_second": 1.209
+    "train_loss": 3.7704881070840237,
+    "train_runtime": 13409.8958,
+    "train_samples_per_second": 42.504,
+    "train_steps_per_second": 0.664
 }

trainer_state.json

@@ -1,5 +1,5 @@
 {
-  "best_metric": 3.185894727706909,
+  "best_metric": 3.1860642433166504,
   "best_model_checkpoint": "./output/models/parallel-mean-bottleneck-gpt2-medium-wikitext/checkpoint-8500",
   "epoch": 5.0,
   "eval_steps": 500,
@@ -10,822 +10,822 @@
   "log_history": [
     {
       "epoch": 0.05611672278338945,
-      "grad_norm": 1.5436944961547852,
+      "grad_norm": 1.54179847240448,
       "learning_rate": 1.1223344556677892e-05,
       "loss": 8.933,
       "step": 100
     },
     {
       "epoch": 0.1122334455667789,
-      "grad_norm": 1.0424357652664185,
+      "grad_norm": 1.036051869392395,
       "learning_rate": 2.2446689113355783e-05,
       "loss": 7.3249,
       "step": 200
     },
     {
       "epoch": 0.16835016835016836,
-      "grad_norm": 1.080315351486206,
+      "grad_norm": 1.0635498762130737,
       "learning_rate": 3.3670033670033675e-05,
       "loss": 6.6032,
       "step": 300
     },
     {
       "epoch": 0.2244668911335578,
-      "grad_norm": 1.2949663400650024,
+      "grad_norm": 1.8017141819000244,
       "learning_rate": 4.4893378226711566e-05,
-      "loss": 6.2747,
+      "loss": 6.276,
       "step": 400
     },
     {
       "epoch": 0.28058361391694725,
-      "grad_norm": 1.1373127698898315,
+      "grad_norm": 1.3540623188018799,
       "learning_rate": 5.611672278338945e-05,
-      "loss": 6.0432,
+      "loss": 6.0438,
       "step": 500
     },
     {
       "epoch": 0.28058361391694725,
-      "eval_accuracy": 0.1909473862768935,
-      "eval_bleu": 0.03783860296240099,
-      "eval_loss": 5.917980670928955,
-      "eval_perplexity": 371.66045085486866,
-      "eval_runtime": 11.0079,
-      "eval_samples_per_second": 103.653,
-      "eval_steps_per_second": 1.635,
+      "eval_accuracy": 0.189721420475428,
+      "eval_bleu": 0.03593862692187448,
+      "eval_loss": 5.919970989227295,
+      "eval_perplexity": 372.40091008131225,
+      "eval_runtime": 11.0176,
+      "eval_samples_per_second": 103.562,
+      "eval_steps_per_second": 1.634,
       "step": 500
     },
     {
       "epoch": 0.3367003367003367,
-      "grad_norm": 1.0700047016143799,
+      "grad_norm": 1.158894658088684,
       "learning_rate": 6.734006734006735e-05,
-      "loss": 5.8439,
+      "loss": 5.8447,
       "step": 600
     },
     {
       "epoch": 0.39281705948372614,
-      "grad_norm": 1.4188077449798584,
+      "grad_norm": 1.1720908880233765,
       "learning_rate": 7.856341189674523e-05,
-      "loss": 5.6426,
+      "loss": 5.6413,
       "step": 700
     },
     {
       "epoch": 0.4489337822671156,
-      "grad_norm": 1.1133205890655518,
+      "grad_norm": 1.2613935470581055,
       "learning_rate": 8.978675645342313e-05,
-      "loss": 5.464,
+      "loss": 5.4607,
       "step": 800
     },
     {
       "epoch": 0.5050505050505051,
-      "grad_norm": 1.3167998790740967,
+      "grad_norm": 1.0967986583709717,
       "learning_rate": 9.988776655443322e-05,
-      "loss": 5.25,
+      "loss": 5.2463,
       "step": 900
     },
     {
       "epoch": 0.5611672278338945,
-      "grad_norm": 1.0362534523010254,
+      "grad_norm": 1.0036829710006714,
       "learning_rate": 9.864072827035791e-05,
-      "loss": 5.0476,
+      "loss": 5.0422,
       "step": 1000
     },
     {
       "epoch": 0.5611672278338945,
-      "eval_accuracy": 0.26329735967574874,
-      "eval_bleu": 0.06118160815453846,
-      "eval_loss": 4.8985185623168945,
-      "eval_perplexity": 134.0909850321564,
-      "eval_runtime": 11.0771,
-      "eval_samples_per_second": 103.006,
-      "eval_steps_per_second": 1.625,
+      "eval_accuracy": 0.2635603726045048,
+      "eval_bleu": 0.06103039117966378,
+      "eval_loss": 4.893420696258545,
+      "eval_perplexity": 133.40914658800884,
+      "eval_runtime": 11.0697,
+      "eval_samples_per_second": 103.074,
+      "eval_steps_per_second": 1.626,
       "step": 1000
     },
     {
       "epoch": 0.6172839506172839,
-      "grad_norm": 0.9963632225990295,
+      "grad_norm": 0.9532070159912109,
       "learning_rate": 9.73936899862826e-05,
-      "loss": 4.8705,
+      "loss": 4.8675,
       "step": 1100
     },
     {
       "epoch": 0.6734006734006734,
-      "grad_norm": 0.9135327935218811,
+      "grad_norm": 0.9311050772666931,
       "learning_rate": 9.614665170220725e-05,
-      "loss": 4.7183,
+      "loss": 4.7157,
       "step": 1200
     },
     {
       "epoch": 0.7295173961840629,
-      "grad_norm": 0.9141886830329895,
+      "grad_norm": 0.9085267782211304,
       "learning_rate": 9.489961341813194e-05,
-      "loss": 4.5877,
+      "loss": 4.5856,
       "step": 1300
     },
     {
       "epoch": 0.7856341189674523,
-      "grad_norm": 0.8190032839775085,
+      "grad_norm": 0.8684507608413696,
       "learning_rate": 9.365257513405662e-05,
-      "loss": 4.4743,
+      "loss": 4.4711,
       "step": 1400
     },
     {
       "epoch": 0.8417508417508418,
-      "grad_norm": 0.8178410530090332,
+      "grad_norm": 0.8369265794754028,
       "learning_rate": 9.24055368499813e-05,
-      "loss": 4.3528,
+      "loss": 4.3494,
       "step": 1500
     },
     {
       "epoch": 0.8417508417508418,
-      "eval_accuracy": 0.3181762495041735,
-      "eval_bleu": 0.08335013046001274,
-      "eval_loss": 4.239789962768555,
-      "eval_perplexity": 69.39327513659308,
-      "eval_runtime": 11.1029,
-      "eval_samples_per_second": 102.766,
-      "eval_steps_per_second": 1.621,
+      "eval_accuracy": 0.318285052898154,
+      "eval_bleu": 0.08333640625132697,
+      "eval_loss": 4.238930702209473,
+      "eval_perplexity": 69.33367384239048,
+      "eval_runtime": 11.0928,
+      "eval_samples_per_second": 102.859,
+      "eval_steps_per_second": 1.623,
       "step": 1500
     },
     {
       "epoch": 0.8978675645342312,
-      "grad_norm": 0.9094557166099548,
+      "grad_norm": 0.882328987121582,
       "learning_rate": 9.115849856590598e-05,
-      "loss": 4.2651,
+      "loss": 4.2632,
       "step": 1600
     },
     {
       "epoch": 0.9539842873176206,
-      "grad_norm": 0.8006865382194519,
+      "grad_norm": 0.8303613662719727,
       "learning_rate": 8.991146028183066e-05,
-      "loss": 4.1573,
+      "loss": 4.1548,
       "step": 1700
     },
     {
       "epoch": 1.0101010101010102,
-      "grad_norm": 0.8540999293327332,
+      "grad_norm": 0.8696436285972595,
       "learning_rate": 8.866442199775533e-05,
-      "loss": 4.0896,
+      "loss": 4.0877,
       "step": 1800
     },
     {
       "epoch": 1.0662177328843996,
-      "grad_norm": 0.8293948173522949,
+      "grad_norm": 0.8291804194450378,
       "learning_rate": 8.741738371368002e-05,
-      "loss": 4.0039,
+      "loss": 4.0032,
       "step": 1900
     },
     {
       "epoch": 1.122334455667789,
-      "grad_norm": 0.7962952256202698,
+      "grad_norm": 0.7963048815727234,
       "learning_rate": 8.617034542960469e-05,
-      "loss": 3.9497,
+      "loss": 3.9486,
       "step": 2000
     },
     {
       "epoch": 1.122334455667789,
-      "eval_accuracy": 0.35195670481639213,
-      "eval_bleu": 0.10543076899138493,
-      "eval_loss": 3.8878896236419678,
-      "eval_perplexity": 48.807774985506946,
-      "eval_runtime": 11.1215,
-      "eval_samples_per_second": 102.594,
-      "eval_steps_per_second": 1.618,
+      "eval_accuracy": 0.352146039856311,
+      "eval_bleu": 0.10372829354298996,
+      "eval_loss": 3.885582208633423,
+      "eval_perplexity": 48.695285023394725,
+      "eval_runtime": 11.0694,
+      "eval_samples_per_second": 103.077,
+      "eval_steps_per_second": 1.626,
       "step": 2000
     },
     {
       "epoch": 1.1784511784511784,
-      "grad_norm": 0.7807645797729492,
+      "grad_norm": 0.7901706099510193,
       "learning_rate": 8.492330714552937e-05,
-      "loss": 3.9058,
+      "loss": 3.9051,
       "step": 2100
     },
     {
       "epoch": 1.2345679012345678,
-      "grad_norm": 0.7973741292953491,
+      "grad_norm": 0.8067757487297058,
       "learning_rate": 8.367626886145406e-05,
-      "loss": 3.859,
+      "loss": 3.8583,
       "step": 2200
     },
     {
       "epoch": 1.2906846240179575,
-      "grad_norm": 0.7875047922134399,
+      "grad_norm": 0.7848672866821289,
       "learning_rate": 8.242923057737873e-05,
-      "loss": 3.84,
+      "loss": 3.8389,
       "step": 2300
     },
     {
       "epoch": 1.3468013468013469,
-      "grad_norm": 0.7630696296691895,
+      "grad_norm": 0.7620055675506592,
       "learning_rate": 8.11821922933034e-05,
-      "loss": 3.7989,
+      "loss": 3.7985,
       "step": 2400
     },
     {
       "epoch": 1.4029180695847363,
-      "grad_norm": 0.7279968857765198,
+      "grad_norm": 0.7321527600288391,
       "learning_rate": 7.993515400922809e-05,
-      "loss": 3.7614,
+      "loss": 3.7605,
       "step": 2500
     },
     {
       "epoch": 1.4029180695847363,
-      "eval_accuracy": 0.367412783799089,
-      "eval_bleu": 0.12071702163720974,
-      "eval_loss": 3.7127673625946045,
-      "eval_perplexity": 40.967020402690565,
-      "eval_runtime": 11.2307,
-      "eval_samples_per_second": 101.596,
-      "eval_steps_per_second": 1.603,
+      "eval_accuracy": 0.36705039139236645,
+      "eval_bleu": 0.12056364590808125,
+      "eval_loss": 3.7143070697784424,
+      "eval_perplexity": 41.03014620345847,
+      "eval_runtime": 11.0834,
+      "eval_samples_per_second": 102.947,
+      "eval_steps_per_second": 1.624,
       "step": 2500
     },
     {
       "epoch": 1.4590347923681257,
-      "grad_norm": 0.7240808606147766,
+      "grad_norm": 0.7496013641357422,
       "learning_rate": 7.868811572515277e-05,
-      "loss": 3.7429,
+      "loss": 3.7426,
       "step": 2600
     },
     {
       "epoch": 1.5151515151515151,
-      "grad_norm": 0.7098649144172668,
+      "grad_norm": 0.7082226872444153,
       "learning_rate": 7.744107744107744e-05,
-      "loss": 3.7126,
+      "loss": 3.7123,
       "step": 2700
     },
     {
       "epoch": 1.5712682379349046,
-      "grad_norm": 0.7229514122009277,
+      "grad_norm": 0.7311062812805176,
       "learning_rate": 7.619403915700213e-05,
       "loss": 3.6816,
       "step": 2800
     },
     {
       "epoch": 1.627384960718294,
-      "grad_norm": 0.7334359288215637,
+      "grad_norm": 0.726064920425415,
       "learning_rate": 7.49470008729268e-05,
-      "loss": 3.6683,
+      "loss": 3.6676,
       "step": 2900
     },
     {
       "epoch": 1.6835016835016834,
-      "grad_norm": 0.7653447389602661,
+      "grad_norm": 0.7564496994018555,
       "learning_rate": 7.369996258885148e-05,
-      "loss": 3.6543,
+      "loss": 3.6544,
       "step": 3000
     },
     {
       "epoch": 1.6835016835016834,
-      "eval_accuracy": 0.37795900253846026,
-      "eval_bleu": 0.13095688045902315,
-      "eval_loss": 3.590174913406372,
-      "eval_perplexity": 36.24041430644117,
-      "eval_runtime": 11.0101,
-      "eval_samples_per_second": 103.632,
-      "eval_steps_per_second": 1.635,
+      "eval_accuracy": 0.37813634350345215,
+      "eval_bleu": 0.13317535043978146,
+      "eval_loss": 3.589838981628418,
+      "eval_perplexity": 36.22824204426872,
+      "eval_runtime": 11.0756,
+      "eval_samples_per_second": 103.019,
+      "eval_steps_per_second": 1.625,
       "step": 3000
     },
     {
       "epoch": 1.7396184062850728,
-      "grad_norm": 0.7163519263267517,
+      "grad_norm": 0.7231422066688538,
       "learning_rate": 7.245292430477615e-05,
-      "loss": 3.621,
+      "loss": 3.6204,
       "step": 3100
     },
     {
       "epoch": 1.7957351290684624,
-      "grad_norm": 0.7292365431785583,
+      "grad_norm": 0.7412242293357849,
       "learning_rate": 7.120588602070084e-05,
-      "loss": 3.6054,
+      "loss": 3.605,
       "step": 3200
     },
     {
       "epoch": 1.8518518518518519,
-      "grad_norm": 0.7313345670700073,
+      "grad_norm": 0.7535395622253418,
       "learning_rate": 6.995884773662552e-05,
-      "loss": 3.578,
+      "loss": 3.5779,
       "step": 3300
     },
     {
       "epoch": 1.9079685746352413,
-      "grad_norm": 0.7751216292381287,
+      "grad_norm": 0.7545406222343445,
       "learning_rate": 6.871180945255021e-05,
-      "loss": 3.5709,
+      "loss": 3.571,
       "step": 3400
     },
     {
       "epoch": 1.964085297418631,
-      "grad_norm": 0.7018395066261292,
+      "grad_norm": 0.716988742351532,
       "learning_rate": 6.746477116847487e-05,
       "loss": 3.5527,
       "step": 3500
     },
     {
       "epoch": 1.964085297418631,
-      "eval_accuracy": 0.38635314154807526,
-      "eval_bleu": 0.13373604633044894,
-      "eval_loss": 3.504826545715332,
-      "eval_perplexity": 33.2756715439979,
-      "eval_runtime": 11.1003,
-      "eval_samples_per_second": 102.79,
-      "eval_steps_per_second": 1.622,
+      "eval_accuracy": 0.3861560960314176,
+      "eval_bleu": 0.13491971280952403,
+      "eval_loss": 3.505063772201538,
+      "eval_perplexity": 33.283566351026074,
+      "eval_runtime": 11.0804,
+      "eval_samples_per_second": 102.974,
+      "eval_steps_per_second": 1.624,
       "step": 3500
     },
     {
       "epoch": 2.0202020202020203,
-      "grad_norm": 0.7037595510482788,
+      "grad_norm": 0.6990429759025574,
       "learning_rate": 6.621773288439955e-05,
-      "loss": 3.5173,
+      "loss": 3.517,
       "step": 3600
     },
     {
       "epoch": 2.0763187429854097,
-      "grad_norm": 0.745313286781311,
+      "grad_norm": 0.7404659390449524,
       "learning_rate": 6.497069460032424e-05,
-      "loss": 3.4633,
+      "loss": 3.4636,
       "step": 3700
     },
     {
       "epoch": 2.132435465768799,
-      "grad_norm": 0.7386572360992432,
+      "grad_norm": 0.7441074848175049,
       "learning_rate": 6.372365631624892e-05,
-      "loss": 3.4408,
+      "loss": 3.4411,
       "step": 3800
     },
     {
       "epoch": 2.1885521885521886,
-      "grad_norm": 0.709396243095398,
+      "grad_norm": 0.701506495475769,
       "learning_rate": 6.247661803217359e-05,
       "loss": 3.4414,
       "step": 3900
     },
     {
       "epoch": 2.244668911335578,
-      "grad_norm": 0.7127860188484192,
+      "grad_norm": 0.7056393027305603,
       "learning_rate": 6.122957974809826e-05,
-      "loss": 3.4348,
+      "loss": 3.4346,
       "step": 4000
     },
     {
       "epoch": 2.244668911335578,
-      "eval_accuracy": 0.39228335487983224,
-      "eval_bleu": 0.13612405667589095,
-      "eval_loss": 3.4400899410247803,
-      "eval_perplexity": 31.189763281432384,
-      "eval_runtime": 11.0954,
-      "eval_samples_per_second": 102.835,
-      "eval_steps_per_second": 1.622,
+      "eval_accuracy": 0.39188926384651696,
+      "eval_bleu": 0.1334943569993854,
+      "eval_loss": 3.4409983158111572,
+      "eval_perplexity": 31.218108147922273,
+      "eval_runtime": 11.0871,
+      "eval_samples_per_second": 102.912,
+      "eval_steps_per_second": 1.624,
       "step": 4000
     },
     {
       "epoch": 2.3007856341189674,
-      "grad_norm": 0.7178986668586731,
+      "grad_norm": 0.7123896479606628,
       "learning_rate": 5.998254146402295e-05,
-      "loss": 3.4225,
+      "loss": 3.4223,
       "step": 4100
     },
     {
       "epoch": 2.356902356902357,
-      "grad_norm": 0.701989471912384,
+      "grad_norm": 0.7008007168769836,
       "learning_rate": 5.8735503179947625e-05,
-      "loss": 3.4018,
+      "loss": 3.4019,
       "step": 4200
     },
     {
       "epoch": 2.4130190796857462,
-      "grad_norm": 0.7325447797775269,
+      "grad_norm": 0.7202064394950867,
       "learning_rate": 5.748846489587231e-05,
       "loss": 3.3989,
       "step": 4300
     },
     {
       "epoch": 2.4691358024691357,
-      "grad_norm": 0.7156170010566711,
+      "grad_norm": 0.7087401151657104,
       "learning_rate": 5.624142661179699e-05,
-      "loss": 3.3827,
+      "loss": 3.3828,
       "step": 4400
     },
     {
       "epoch": 2.525252525252525,
-      "grad_norm": 0.7117893099784851,
+      "grad_norm": 0.7096906304359436,
       "learning_rate": 5.4994388327721666e-05,
-      "loss": 3.3739,
+      "loss": 3.374,
       "step": 4500
     },
     {
       "epoch": 2.525252525252525,
-      "eval_accuracy": 0.3973971143969165,
-      "eval_bleu": 0.1418527614095323,
-      "eval_loss": 3.3868210315704346,
-      "eval_perplexity": 29.57179488564003,
-      "eval_runtime": 10.9961,
-      "eval_samples_per_second": 103.765,
-      "eval_steps_per_second": 1.637,
+      "eval_accuracy": 0.3972454750210539,
+      "eval_bleu": 0.13536753244366495,
+      "eval_loss": 3.3866658210754395,
+      "eval_perplexity": 29.56720538889618,
+      "eval_runtime": 11.1089,
+      "eval_samples_per_second": 102.71,
+      "eval_steps_per_second": 1.62,
       "step": 4500
     },
     {
       "epoch": 2.581369248035915,
-      "grad_norm": 0.7047144770622253,
+      "grad_norm": 0.7192590832710266,
       "learning_rate": 5.374735004364634e-05,
-      "loss": 3.3787,
+      "loss": 3.3785,
       "step": 4600
     },
     {
       "epoch": 2.637485970819304,
-      "grad_norm": 0.6755483746528625,
+      "grad_norm": 0.6848036646842957,
       "learning_rate": 5.250031175957102e-05,
-      "loss": 3.3641,
+      "loss": 3.3642,
       "step": 4700
     },
     {
       "epoch": 2.6936026936026938,
-      "grad_norm": 0.7206361889839172,
+      "grad_norm": 0.7250380516052246,
       "learning_rate": 5.12532734754957e-05,
       "loss": 3.3589,
       "step": 4800
     },
     {
       "epoch": 2.749719416386083,
-      "grad_norm": 0.6900231838226318,
+      "grad_norm": 0.6998625993728638,
       "learning_rate": 5.000623519142038e-05,
-      "loss": 3.3492,
+      "loss": 3.3491,
       "step": 4900
     },
     {
       "epoch": 2.8058361391694726,
-      "grad_norm": 0.7102543115615845,
+      "grad_norm": 0.7044927477836609,
       "learning_rate": 4.8759196907345056e-05,
-      "loss": 3.3441,
+      "loss": 3.3442,
       "step": 5000
     },
     {
       "epoch": 2.8058361391694726,
-      "eval_accuracy": 0.4019574330280841,
-      "eval_bleu": 0.13938720153734416,
-      "eval_loss": 3.3418636322021484,
-      "eval_perplexity": 28.271765813386498,
-      "eval_runtime": 11.1267,
-      "eval_samples_per_second": 102.546,
-      "eval_steps_per_second": 1.618,
+      "eval_accuracy": 0.40168071258512583,
+      "eval_bleu": 0.14053869067204397,
+      "eval_loss": 3.340980052947998,
+      "eval_perplexity": 28.24679650044353,
+      "eval_runtime": 11.0994,
+      "eval_samples_per_second": 102.798,
+      "eval_steps_per_second": 1.622,
       "step": 5000
     },
     {
       "epoch": 2.861952861952862,
-      "grad_norm": 0.7073910236358643,
+      "grad_norm": 0.7032910585403442,
       "learning_rate": 4.751215862326974e-05,
-      "loss": 3.3419,
+      "loss": 3.342,
       "step": 5100
     },
     {
       "epoch": 2.9180695847362514,
-      "grad_norm": 0.6984754800796509,
+      "grad_norm": 0.6985570788383484,
       "learning_rate": 4.626512033919442e-05,
-      "loss": 3.3203,
+      "loss": 3.3205,
       "step": 5200
     },
     {
       "epoch": 2.974186307519641,
-      "grad_norm": 0.6974130272865295,
+      "grad_norm": 0.7079312801361084,
       "learning_rate": 4.5018082055119096e-05,
-      "loss": 3.3243,
+      "loss": 3.3244,
       "step": 5300
     },
     {
       "epoch": 3.0303030303030303,
-      "grad_norm": 0.7236476540565491,
+      "grad_norm": 0.7505896687507629,
       "learning_rate": 4.3771043771043774e-05,
-      "loss": 3.2725,
+      "loss": 3.2727,
       "step": 5400
     },
     {
       "epoch": 3.0864197530864197,
-      "grad_norm": 0.7239139080047607,
+      "grad_norm": 0.7251821756362915,
       "learning_rate": 4.252400548696845e-05,
-      "loss": 3.2252,
+      "loss": 3.2251,
       "step": 5500
     },
     {
       "epoch": 3.0864197530864197,
-      "eval_accuracy": 0.4057432771068235,
-      "eval_bleu": 0.1432212835687173,
-      "eval_loss": 3.306666374206543,
-      "eval_perplexity": 27.29398570860935,
-      "eval_runtime": 11.1335,
-      "eval_samples_per_second": 102.483,
-      "eval_steps_per_second": 1.617,
+      "eval_accuracy": 0.40551710312248607,
+      "eval_bleu": 0.14035240790883957,
+      "eval_loss": 3.3072268962860107,
+      "eval_perplexity": 27.3092888787174,
+      "eval_runtime": 11.1046,
+      "eval_samples_per_second": 102.75,
+      "eval_steps_per_second": 1.621,
       "step": 5500
     },
     {
       "epoch": 3.142536475869809,
-      "grad_norm": 0.737918496131897,
+      "grad_norm": 0.7410485744476318,
       "learning_rate": 4.127696720289313e-05,
-      "loss": 3.2316,
+      "loss": 3.2314,
       "step": 5600
     },
     {
       "epoch": 3.1986531986531985,
-      "grad_norm": 0.7476251125335693,
+      "grad_norm": 0.7442547678947449,
       "learning_rate": 4.002992891881781e-05,
-      "loss": 3.2276,
+      "loss": 3.2277,
       "step": 5700
     },
     {
       "epoch": 3.254769921436588,
-      "grad_norm": 0.7400563359260559,
+      "grad_norm": 0.7330621480941772,
       "learning_rate": 3.8782890634742486e-05,
-      "loss": 3.224,
+      "loss": 3.2246,
       "step": 5800
     },
     {
       "epoch": 3.3108866442199774,
-      "grad_norm": 0.7361284494400024,
+      "grad_norm": 0.7348044514656067,
       "learning_rate": 3.7535852350667164e-05,
-      "loss": 3.2252,
+      "loss": 3.2255,
       "step": 5900
     },
     {
       "epoch": 3.3670033670033668,
-      "grad_norm": 0.7335928082466125,
+      "grad_norm": 0.7259684801101685,
       "learning_rate": 3.628881406659185e-05,
-      "loss": 3.2188,
+      "loss": 3.2187,
       "step": 6000
     },
     {
       "epoch": 3.3670033670033668,
-      "eval_accuracy": 0.4087632138295111,
-      "eval_bleu": 0.14205763499672933,
-      "eval_loss": 3.2775487899780273,
-      "eval_perplexity": 26.510709673638566,
-      "eval_runtime": 11.1182,
-      "eval_samples_per_second": 102.625,
-      "eval_steps_per_second": 1.619,
+      "eval_accuracy": 0.40877263774552514,
+      "eval_bleu": 0.14006482323692013,
+      "eval_loss": 3.278057813644409,
+      "eval_perplexity": 26.524207687387797,
+      "eval_runtime": 11.0977,
+      "eval_samples_per_second": 102.814,
+      "eval_steps_per_second": 1.622,
       "step": 6000
     },
     {
       "epoch": 3.4231200897867566,
-      "grad_norm": 0.7380982637405396,
+      "grad_norm": 0.7419958710670471,
       "learning_rate": 3.504177578251652e-05,
-      "loss": 3.1985,
+      "loss": 3.1992,
       "step": 6100
     },
     {
       "epoch": 3.479236812570146,
-      "grad_norm": 0.7403559684753418,
+      "grad_norm": 0.7455360889434814,
       "learning_rate": 3.3794737498441205e-05,
-      "loss": 3.2097,
+      "loss": 3.2099,
       "step": 6200
     },
     {
       "epoch": 3.5353535353535355,
-      "grad_norm": 0.7356846928596497,
+      "grad_norm": 0.7269027829170227,
       "learning_rate": 3.254769921436588e-05,
-      "loss": 3.1995,
+      "loss": 3.1998,
       "step": 6300
     },
     {
       "epoch": 3.591470258136925,
-      "grad_norm": 0.731614887714386,
+      "grad_norm": 0.7311801910400391,
       "learning_rate": 3.130066093029056e-05,
-      "loss": 3.201,
+      "loss": 3.2011,
       "step": 6400
     },
     {
       "epoch": 3.6475869809203143,
-      "grad_norm": 0.7287305593490601,
+      "grad_norm": 0.7334641218185425,
       "learning_rate": 3.0053622646215242e-05,
-      "loss": 3.1971,
+      "loss": 3.1975,
       "step": 6500
     },
     {
       "epoch": 3.6475869809203143,
-      "eval_accuracy": 0.4115484093714848,
-      "eval_bleu": 0.14263540857108656,
-      "eval_loss": 3.250213146209717,
-      "eval_perplexity": 25.795837616279766,
-      "eval_runtime": 11.1156,
-      "eval_samples_per_second": 102.648,
-      "eval_steps_per_second": 1.619,
+      "eval_accuracy": 0.41177629679509753,
+      "eval_bleu": 0.14331769757778526,
+      "eval_loss": 3.2494168281555176,
+      "eval_perplexity": 25.775304101751964,
+      "eval_runtime": 11.123,
+      "eval_samples_per_second": 102.581,
+      "eval_steps_per_second": 1.618,
       "step": 6500
     },
     {
       "epoch": 3.7037037037037037,
-      "grad_norm": 0.730910062789917,
+      "grad_norm": 0.7255159616470337,
       "learning_rate": 2.880658436213992e-05,
-      "loss": 3.1895,
+      "loss": 3.1898,
       "step": 6600
     },
     {
       "epoch": 3.759820426487093,
-      "grad_norm": 0.7339698672294617,
+      "grad_norm": 0.730241060256958,
       "learning_rate": 2.7559546078064598e-05,
-      "loss": 3.1804,
+      "loss": 3.1806,
       "step": 6700
     },
     {
       "epoch": 3.8159371492704826,
-      "grad_norm": 0.7487326860427856,
+      "grad_norm": 0.7587730884552002,
       "learning_rate": 2.6312507793989276e-05,
-      "loss": 3.171,
+      "loss": 3.1713,
       "step": 6800
     },
     {
       "epoch": 3.872053872053872,
-      "grad_norm": 0.7359170913696289,
+      "grad_norm": 0.7403008937835693,
       "learning_rate": 2.5065469509913957e-05,
-      "loss": 3.1629,
+      "loss": 3.1627,
       "step": 6900
     },
     {
       "epoch": 3.9281705948372614,
-      "grad_norm": 0.7311303019523621,
+      "grad_norm": 0.7402880191802979,
       "learning_rate": 2.3818431225838632e-05,
-      "loss": 3.1722,
+      "loss": 3.172,
       "step": 7000
     },
     {
       "epoch": 3.9281705948372614,
-      "eval_accuracy": 0.41430276300650337,
-      "eval_bleu": 0.14455082637881378,
-      "eval_loss": 3.2265915870666504,
-      "eval_perplexity": 25.193640134914105,
-      "eval_runtime": 11.1079,
-      "eval_samples_per_second": 102.719,
-      "eval_steps_per_second": 1.62,
+      "eval_accuracy": 0.41417168490194417,
+      "eval_bleu": 0.14449203518513976,
+      "eval_loss": 3.227550506591797,
+      "eval_perplexity": 25.21781039516413,
+      "eval_runtime": 11.0586,
+      "eval_samples_per_second": 103.178,
+      "eval_steps_per_second": 1.628,
       "step": 7000
     },
     {
       "epoch": 3.984287317620651,
-      "grad_norm": 0.740738034248352,
+      "grad_norm": 0.7368999123573303,
       "learning_rate": 2.2571392941763313e-05,
-      "loss": 3.1681,
+      "loss": 3.1682,
       "step": 7100
     },
     {
       "epoch": 4.040404040404041,
-      "grad_norm": 0.7563683986663818,
+      "grad_norm": 0.757147490978241,
       "learning_rate": 2.132435465768799e-05,
-      "loss": 3.1033,
+      "loss": 3.1035,
       "step": 7200
     },
     {
       "epoch": 4.09652076318743,
-      "grad_norm": 0.7448651194572449,
+      "grad_norm": 0.7439301609992981,
       "learning_rate": 2.007731637361267e-05,
-      "loss": 3.0994,
+      "loss": 3.0995,
       "step": 7300
     },
     {
       "epoch": 4.1526374859708195,
-      "grad_norm": 0.7695390582084656,
+      "grad_norm": 0.7797232866287231,
       "learning_rate": 1.883027808953735e-05,
-      "loss": 3.1011,
+      "loss": 3.1013,
       "step": 7400
     },
     {
       "epoch": 4.2087542087542085,
-      "grad_norm": 0.7607565522193909,
+      "grad_norm": 0.7685579657554626,
       "learning_rate": 1.758323980546203e-05,
-      "loss": 3.1052,
+      "loss": 3.1055,
       "step": 7500
     },
     {
       "epoch": 4.2087542087542085,
-      "eval_accuracy": 0.41628521224800663,
-      "eval_bleu": 0.14328695475901032,
-      "eval_loss": 3.210294723510742,
-      "eval_perplexity": 24.78639028015837,
-      "eval_runtime": 11.0972,
-      "eval_samples_per_second": 102.819,
-      "eval_steps_per_second": 1.622,
+      "eval_accuracy": 0.41630834367822295,
+      "eval_bleu": 0.1447371014434901,
+      "eval_loss": 3.2109010219573975,
+      "eval_perplexity": 24.801422786715616,
+      "eval_runtime": 11.144,
+      "eval_samples_per_second": 102.387,
+      "eval_steps_per_second": 1.615,
       "step": 7500
     },
     {
       "epoch": 4.264870931537598,
-      "grad_norm": 0.766854465007782,
+      "grad_norm": 0.7712327241897583,
       "learning_rate": 1.6336201521386706e-05,
       "loss": 3.0911,
       "step": 7600
     },
     {
       "epoch": 4.320987654320987,
-      "grad_norm": 0.7667710185050964,
+      "grad_norm": 0.7560853958129883,
       "learning_rate": 1.5089163237311384e-05,
-      "loss": 3.0779,
+      "loss": 3.0781,
       "step": 7700
     },
     {
       "epoch": 4.377104377104377,
-      "grad_norm": 0.7539135813713074,
+      "grad_norm": 0.7597346901893616,
       "learning_rate": 1.3842124953236066e-05,
-      "loss": 3.0881,
+      "loss": 3.0883,
       "step": 7800
     },
     {
       "epoch": 4.433221099887767,
-      "grad_norm": 0.768229603767395,
+      "grad_norm": 0.7666236162185669,
       "learning_rate": 1.2595086669160744e-05,
-      "loss": 3.0859,
+      "loss": 3.0862,
       "step": 7900
     },
     {
       "epoch": 4.489337822671156,
-      "grad_norm": 0.7591537833213806,
+      "grad_norm": 0.7614879608154297,
       "learning_rate": 1.1348048385085423e-05,
-      "loss": 3.0672,
+      "loss": 3.0676,
       "step": 8000
     },
     {
       "epoch": 4.489337822671156,
-      "eval_accuracy": 0.41795752898068356,
-      "eval_bleu": 0.1437527142843717,
-      "eval_loss": 3.196687936782837,
-      "eval_perplexity": 24.45141131580561,
-      "eval_runtime": 11.1317,
-      "eval_samples_per_second": 102.5,
-      "eval_steps_per_second": 1.617,
+      "eval_accuracy": 0.4178333046332255,
+      "eval_bleu": 0.14527019621911616,
+      "eval_loss": 3.197705030441284,
+      "eval_perplexity": 24.476293342725903,
+      "eval_runtime": 11.0951,
+      "eval_samples_per_second": 102.838,
+      "eval_steps_per_second": 1.622,
       "step": 8000
     },
     {
       "epoch": 4.545454545454545,
-      "grad_norm": 0.7729194760322571,
+      "grad_norm": 0.7722211480140686,
       "learning_rate": 1.0101010101010101e-05,
-      "loss": 3.0826,
+      "loss": 3.0831,
       "step": 8100
     },
     {
       "epoch": 4.601571268237935,
-      "grad_norm": 0.7760916948318481,
+      "grad_norm": 0.7750692963600159,
       "learning_rate": 8.853971816934781e-06,
-      "loss": 3.0755,
+      "loss": 3.0761,
       "step": 8200
     },
     {
       "epoch": 4.657687991021325,
-      "grad_norm": 0.7464035153388977,
+      "grad_norm": 0.7448268532752991,
       "learning_rate": 7.606933532859459e-06,
-      "loss": 3.0829,
+      "loss": 3.0831,
       "step": 8300
     },
     {
       "epoch": 4.713804713804714,
-      "grad_norm": 0.7666931748390198,
+      "grad_norm": 0.7668105959892273,
       "learning_rate": 6.359895248784138e-06,
-      "loss": 3.0744,
+      "loss": 3.0747,
       "step": 8400
     },
     {
       "epoch": 4.7699214365881035,
-      "grad_norm": 0.7630265355110168,
+      "grad_norm": 0.768974244594574,
       "learning_rate": 5.112856964708817e-06,
-      "loss": 3.0774,
+      "loss": 3.0779,
       "step": 8500
     },
     {
       "epoch": 4.7699214365881035,
-      "eval_accuracy": 0.41937625670061845,
-      "eval_bleu": 0.14605654521068695,
-      "eval_loss": 3.185894727706909,
-      "eval_perplexity": 24.18892122468071,
-      "eval_runtime": 11.0781,
-      "eval_samples_per_second": 102.996,
-      "eval_steps_per_second": 1.625,
+      "eval_accuracy": 0.41932913712054815,
+      "eval_bleu": 0.1439933070551526,
+      "eval_loss": 3.1860642433166504,
+      "eval_perplexity": 24.19302197197161,
+      "eval_runtime": 11.1043,
+      "eval_samples_per_second": 102.753,
+      "eval_steps_per_second": 1.621,
       "step": 8500
     },
     {
       "epoch": 4.8260381593714925,
-      "grad_norm": 0.7553364038467407,
+      "grad_norm": 0.7649372816085815,
       "learning_rate": 3.865818680633495e-06,
-      "loss": 3.0679,
+      "loss": 3.0682,
       "step": 8600
     },
     {
       "epoch": 4.882154882154882,
-      "grad_norm": 0.7630147933959961,
+      "grad_norm": 0.7638269066810608,
       "learning_rate": 2.6187803965581742e-06,
-      "loss": 3.0784,
+      "loss": 3.0788,
       "step": 8700
     },
     {
       "epoch": 4.938271604938271,
-      "grad_norm": 0.7633751034736633,
+      "grad_norm": 0.7597787380218506,
       "learning_rate": 1.3717421124828533e-06,
-      "loss": 3.0748,
+      "loss": 3.0747,
       "step": 8800
     },
     {
       "epoch": 4.994388327721661,
-      "grad_norm": 0.7511401772499084,
+      "grad_norm": 0.7531468272209167,
       "learning_rate": 1.2470382840753213e-07,
-      "loss": 3.0648,
+      "loss": 3.0651,
       "step": 8900
     },
     {
       "epoch": 5.0,
       "step": 8910,
       "total_flos": 1.0586630697202483e+18,
-      "train_loss": 1.7646983184129434,
-      "train_runtime": 7367.504,
-      "train_samples_per_second": 77.363,
-      "train_steps_per_second": 1.209
+      "train_loss": 3.7704881070840237,
+      "train_runtime": 13409.8958,
+      "train_samples_per_second": 42.504,
+      "train_steps_per_second": 0.664
     }
   ],
   "logging_steps": 100,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a3d7d0589deb26966fccca9777312b972822cbcf2baece33a62c79e8ceaa6615
+oid sha256:25a6ddb6db8b6570d677716f3e185e04b016980664f78710bc5a660dee677933
 size 5560