Training in progress, step 21000, checkpoint

last-checkpoint/adapter_config.json

@@ -25,13 +25,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "gate_proj",
     "o_proj",
-    "down_proj",
     "q_proj",
-    "k_proj",
-    "up_proj",
     "v_proj",
-    "gate_proj"
+    "k_proj",
+    "down_proj",
+    "up_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:883db002a6734cb265864ebc120f2884e77f649b30dae4a2302eefca4d4137bb
+oid sha256:76d220f6eb62d357cac204cc6ca10e112229c5ea4f345c3e50d0cabaf1919126
 size 389081912

last-checkpoint/global_step21000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc38d7765eee46a46c1cfb655ca1ec93f1839468d5fa79206d9b09ba7266ce4e
+size 1167094117

last-checkpoint/global_step21000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:094b08b7e988cbbcc8f171789dc77f5095d58244f35d03ba5a4ff32417e44eee
+size 1167094245

last-checkpoint/global_step21000/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3857c1c2fa6aa57562e840fc55be0e457a7e501efdbf0d5d16ac48b30f2a4cb2
+size 1222740115

last-checkpoint/latest

@@ -1 +1 @@
-global_step4600
+global_step21000

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c29caf33459e77925ce566a510af138891ad4476e5bca606a1c814b77305442
+oid sha256:1f188a91a21d29457226edd00f5fcf5c04d693fe2d994fee71c314fba23852b9
 size 14917

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d6eccb59d533bc2d9720c2f64f56a9affd86293f6e5cde1146191b5f72a2bbb
+oid sha256:d782a09a6bb8b2213c345e44b4eebe638d7afd04a217515bee34b8c83445597c
 size 14917

last-checkpoint/special_tokens_map.json

@@ -13,5 +13,11 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<|eot_id|>"
+  "pad_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.6345265190702807,
+  "epoch": 2.8967515001034556,
   "eval_steps": 500,
-  "global_step": 4600,
+  "global_step": 21000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -215,6 +215,744 @@
       "mean_token_accuracy": 0.6035999485850334,
       "num_tokens": 299295385.0,
       "step": 4600
+    },
+    {
+      "epoch": 0.6621146285950755,
+      "grad_norm": 0.2658919095993042,
+      "learning_rate": 1.735466960960133e-05,
+      "loss": 1.7675,
+      "mean_token_accuracy": 0.6052964499592781,
+      "num_tokens": 13006009.0,
+      "step": 4800
+    },
+    {
+      "epoch": 0.6897027381198704,
+      "grad_norm": 0.3054528534412384,
+      "learning_rate": 1.7244309559939303e-05,
+      "loss": 1.7733,
+      "mean_token_accuracy": 0.6038838358223438,
+      "num_tokens": 26022591.0,
+      "step": 5000
+    },
+    {
+      "epoch": 0.7172908476446651,
+      "grad_norm": 0.2557850778102875,
+      "learning_rate": 1.7133949510277282e-05,
+      "loss": 1.7678,
+      "mean_token_accuracy": 0.6045010414719582,
+      "num_tokens": 39031786.0,
+      "step": 5200
+    },
+    {
+      "epoch": 0.74487895716946,
+      "grad_norm": 0.2966762185096741,
+      "learning_rate": 1.702358946061526e-05,
+      "loss": 1.7645,
+      "mean_token_accuracy": 0.6054365809261799,
+      "num_tokens": 52042866.0,
+      "step": 5400
+    },
+    {
+      "epoch": 0.7724670666942548,
+      "grad_norm": 0.2732425630092621,
+      "learning_rate": 1.6913229410953235e-05,
+      "loss": 1.7647,
+      "mean_token_accuracy": 0.6047625686228275,
+      "num_tokens": 65053898.0,
+      "step": 5600
+    },
+    {
+      "epoch": 0.8000551762190495,
+      "grad_norm": 0.27680376172065735,
+      "learning_rate": 1.6802869361291214e-05,
+      "loss": 1.7642,
+      "mean_token_accuracy": 0.6054388484358788,
+      "num_tokens": 78051546.0,
+      "step": 5800
+    },
+    {
+      "epoch": 0.8276432857438444,
+      "grad_norm": 0.2786679267883301,
+      "learning_rate": 1.6692509311629192e-05,
+      "loss": 1.7653,
+      "mean_token_accuracy": 0.6052116820216179,
+      "num_tokens": 91069298.0,
+      "step": 6000
+    },
+    {
+      "epoch": 0.8552313952686392,
+      "grad_norm": 0.2635458707809448,
+      "learning_rate": 1.6582149261967168e-05,
+      "loss": 1.7648,
+      "mean_token_accuracy": 0.6061200417578221,
+      "num_tokens": 104059318.0,
+      "step": 6200
+    },
+    {
+      "epoch": 0.8828195047934341,
+      "grad_norm": 0.2765181362628937,
+      "learning_rate": 1.6471789212305146e-05,
+      "loss": 1.7667,
+      "mean_token_accuracy": 0.6046179200708867,
+      "num_tokens": 117087631.0,
+      "step": 6400
+    },
+    {
+      "epoch": 0.9104076143182288,
+      "grad_norm": 0.28083258867263794,
+      "learning_rate": 1.6361429162643124e-05,
+      "loss": 1.7612,
+      "mean_token_accuracy": 0.6063811640441418,
+      "num_tokens": 130092049.0,
+      "step": 6600
+    },
+    {
+      "epoch": 0.9379957238430237,
+      "grad_norm": 0.2751764953136444,
+      "learning_rate": 1.6251069112981103e-05,
+      "loss": 1.7509,
+      "mean_token_accuracy": 0.6079460391402245,
+      "num_tokens": 143094014.0,
+      "step": 6800
+    },
+    {
+      "epoch": 0.9655838333678185,
+      "grad_norm": 0.2823185920715332,
+      "learning_rate": 1.614070906331908e-05,
+      "loss": 1.7557,
+      "mean_token_accuracy": 0.606988143324852,
+      "num_tokens": 156090713.0,
+      "step": 7000
+    },
+    {
+      "epoch": 0.9931719428926132,
+      "grad_norm": 0.2630833387374878,
+      "learning_rate": 1.603034901365706e-05,
+      "loss": 1.7554,
+      "mean_token_accuracy": 0.6067783555388451,
+      "num_tokens": 169115214.0,
+      "step": 7200
+    },
+    {
+      "epoch": 1.02082902269122,
+      "grad_norm": 0.24643921852111816,
+      "learning_rate": 1.5919988963995035e-05,
+      "loss": 1.7542,
+      "mean_token_accuracy": 0.6083373454146255,
+      "num_tokens": 182136269.0,
+      "step": 7400
+    },
+    {
+      "epoch": 1.048417132216015,
+      "grad_norm": 0.2477455735206604,
+      "learning_rate": 1.5809628914333013e-05,
+      "loss": 1.7316,
+      "mean_token_accuracy": 0.6105767801404,
+      "num_tokens": 195134342.0,
+      "step": 7600
+    },
+    {
+      "epoch": 1.0760052417408097,
+      "grad_norm": 0.2505553066730499,
+      "learning_rate": 1.5699268864670992e-05,
+      "loss": 1.7412,
+      "mean_token_accuracy": 0.6090090696513653,
+      "num_tokens": 208152063.0,
+      "step": 7800
+    },
+    {
+      "epoch": 1.1035933512656044,
+      "grad_norm": 0.25275516510009766,
+      "learning_rate": 1.5588908815008967e-05,
+      "loss": 1.7389,
+      "mean_token_accuracy": 0.6091519020497799,
+      "num_tokens": 221135506.0,
+      "step": 8000
+    },
+    {
+      "epoch": 1.1311814607903994,
+      "grad_norm": 0.256531685590744,
+      "learning_rate": 1.5478548765346945e-05,
+      "loss": 1.738,
+      "mean_token_accuracy": 0.6093683485686779,
+      "num_tokens": 234118407.0,
+      "step": 8200
+    },
+    {
+      "epoch": 1.1587695703151941,
+      "grad_norm": 0.26037585735321045,
+      "learning_rate": 1.5368188715684924e-05,
+      "loss": 1.7338,
+      "mean_token_accuracy": 0.610037949681282,
+      "num_tokens": 247128181.0,
+      "step": 8400
+    },
+    {
+      "epoch": 1.1863576798399889,
+      "grad_norm": 0.2678148150444031,
+      "learning_rate": 1.52578286660229e-05,
+      "loss": 1.7367,
+      "mean_token_accuracy": 0.609309321641922,
+      "num_tokens": 260146123.0,
+      "step": 8600
+    },
+    {
+      "epoch": 1.2139457893647838,
+      "grad_norm": 0.2542251646518707,
+      "learning_rate": 1.5147468616360879e-05,
+      "loss": 1.7395,
+      "mean_token_accuracy": 0.6088805337250233,
+      "num_tokens": 273180867.0,
+      "step": 8800
+    },
+    {
+      "epoch": 1.2415338988895785,
+      "grad_norm": 0.25549325346946716,
+      "learning_rate": 1.5037108566698858e-05,
+      "loss": 1.7278,
+      "mean_token_accuracy": 0.6110722988843917,
+      "num_tokens": 286192947.0,
+      "step": 9000
+    },
+    {
+      "epoch": 1.2691220084143735,
+      "grad_norm": 0.26431170105934143,
+      "learning_rate": 1.4926748517036833e-05,
+      "loss": 1.7293,
+      "mean_token_accuracy": 0.6109644013643265,
+      "num_tokens": 299187058.0,
+      "step": 9200
+    },
+    {
+      "epoch": 1.2967101179391682,
+      "grad_norm": 0.25987866520881653,
+      "learning_rate": 1.4816388467374811e-05,
+      "loss": 1.7362,
+      "mean_token_accuracy": 0.6091528676450253,
+      "num_tokens": 312229168.0,
+      "step": 9400
+    },
+    {
+      "epoch": 1.324298227463963,
+      "grad_norm": 0.2580626308917999,
+      "learning_rate": 1.470602841771279e-05,
+      "loss": 1.7298,
+      "mean_token_accuracy": 0.6105604329705239,
+      "num_tokens": 325252754.0,
+      "step": 9600
+    },
+    {
+      "epoch": 1.3518863369887577,
+      "grad_norm": 0.26252448558807373,
+      "learning_rate": 1.4595668368050766e-05,
+      "loss": 1.7258,
+      "mean_token_accuracy": 0.6110147424042225,
+      "num_tokens": 338265857.0,
+      "step": 9800
+    },
+    {
+      "epoch": 1.3794744465135527,
+      "grad_norm": 0.267373263835907,
+      "learning_rate": 1.4485308318388745e-05,
+      "loss": 1.73,
+      "mean_token_accuracy": 0.6104732736945152,
+      "num_tokens": 351268079.0,
+      "step": 10000
+    },
+    {
+      "epoch": 1.4070625560383474,
+      "grad_norm": 0.2667401134967804,
+      "learning_rate": 1.4374948268726723e-05,
+      "loss": 1.732,
+      "mean_token_accuracy": 0.6098475851118564,
+      "num_tokens": 364302227.0,
+      "step": 10200
+    },
+    {
+      "epoch": 1.4346506655631424,
+      "grad_norm": 0.26889705657958984,
+      "learning_rate": 1.4264588219064698e-05,
+      "loss": 1.731,
+      "mean_token_accuracy": 0.6093322136998176,
+      "num_tokens": 377338727.0,
+      "step": 10400
+    },
+    {
+      "epoch": 1.462238775087937,
+      "grad_norm": 0.2645055055618286,
+      "learning_rate": 1.4154228169402677e-05,
+      "loss": 1.7287,
+      "mean_token_accuracy": 0.6106591558456421,
+      "num_tokens": 390354064.0,
+      "step": 10600
+    },
+    {
+      "epoch": 1.4898268846127318,
+      "grad_norm": 0.268999308347702,
+      "learning_rate": 1.4043868119740655e-05,
+      "loss": 1.7223,
+      "mean_token_accuracy": 0.6116151364147663,
+      "num_tokens": 403371674.0,
+      "step": 10800
+    },
+    {
+      "epoch": 1.5174149941375266,
+      "grad_norm": 0.2641783356666565,
+      "learning_rate": 1.3933508070078632e-05,
+      "loss": 1.7198,
+      "mean_token_accuracy": 0.6120837351679802,
+      "num_tokens": 416381735.0,
+      "step": 11000
+    },
+    {
+      "epoch": 1.5450031036623215,
+      "grad_norm": 0.26654648780822754,
+      "learning_rate": 1.382314802041661e-05,
+      "loss": 1.7272,
+      "mean_token_accuracy": 0.6110439296066761,
+      "num_tokens": 429385525.0,
+      "step": 11200
+    },
+    {
+      "epoch": 1.5725912131871165,
+      "grad_norm": 0.2701609134674072,
+      "learning_rate": 1.3712787970754589e-05,
+      "loss": 1.7308,
+      "mean_token_accuracy": 0.610301011800766,
+      "num_tokens": 442382821.0,
+      "step": 11400
+    },
+    {
+      "epoch": 1.6001793227119112,
+      "grad_norm": 0.26637518405914307,
+      "learning_rate": 1.3602427921092564e-05,
+      "loss": 1.7314,
+      "mean_token_accuracy": 0.6097376590967178,
+      "num_tokens": 455417304.0,
+      "step": 11600
+    },
+    {
+      "epoch": 1.627767432236706,
+      "grad_norm": 0.2647789716720581,
+      "learning_rate": 1.3492067871430543e-05,
+      "loss": 1.7256,
+      "mean_token_accuracy": 0.6110027520358563,
+      "num_tokens": 468439793.0,
+      "step": 11800
+    },
+    {
+      "epoch": 1.6553555417615007,
+      "grad_norm": 0.26624011993408203,
+      "learning_rate": 1.3381707821768521e-05,
+      "loss": 1.7243,
+      "mean_token_accuracy": 0.6117079831659794,
+      "num_tokens": 481425220.0,
+      "step": 12000
+    },
+    {
+      "epoch": 1.6829436512862956,
+      "grad_norm": 0.2641337215900421,
+      "learning_rate": 1.3271347772106498e-05,
+      "loss": 1.718,
+      "mean_token_accuracy": 0.6129114121198654,
+      "num_tokens": 494414024.0,
+      "step": 12200
+    },
+    {
+      "epoch": 1.7105317608110904,
+      "grad_norm": 0.2691522538661957,
+      "learning_rate": 1.3160987722444476e-05,
+      "loss": 1.7262,
+      "mean_token_accuracy": 0.6111969475448131,
+      "num_tokens": 507414476.0,
+      "step": 12400
+    },
+    {
+      "epoch": 1.7381198703358853,
+      "grad_norm": 0.2704949676990509,
+      "learning_rate": 1.3050627672782455e-05,
+      "loss": 1.7212,
+      "mean_token_accuracy": 0.6119842390716076,
+      "num_tokens": 520416862.0,
+      "step": 12600
+    },
+    {
+      "epoch": 1.76570797986068,
+      "grad_norm": 0.2640060782432556,
+      "learning_rate": 1.294026762312043e-05,
+      "loss": 1.7197,
+      "mean_token_accuracy": 0.6121780882775784,
+      "num_tokens": 533414023.0,
+      "step": 12800
+    },
+    {
+      "epoch": 1.7932960893854748,
+      "grad_norm": 0.2656666338443756,
+      "learning_rate": 1.2829907573458408e-05,
+      "loss": 1.7166,
+      "mean_token_accuracy": 0.6125291535258293,
+      "num_tokens": 546421754.0,
+      "step": 13000
+    },
+    {
+      "epoch": 1.8208841989102695,
+      "grad_norm": 0.2837753891944885,
+      "learning_rate": 1.2719547523796387e-05,
+      "loss": 1.7143,
+      "mean_token_accuracy": 0.6129081077873707,
+      "num_tokens": 559432463.0,
+      "step": 13200
+    },
+    {
+      "epoch": 1.8484723084350645,
+      "grad_norm": 0.27201464772224426,
+      "learning_rate": 1.2609187474134364e-05,
+      "loss": 1.7167,
+      "mean_token_accuracy": 0.6130946539342403,
+      "num_tokens": 572437690.0,
+      "step": 13400
+    },
+    {
+      "epoch": 1.8760604179598594,
+      "grad_norm": 0.26749759912490845,
+      "learning_rate": 1.2498827424472342e-05,
+      "loss": 1.7224,
+      "mean_token_accuracy": 0.6114400824904442,
+      "num_tokens": 585452377.0,
+      "step": 13600
+    },
+    {
+      "epoch": 1.9036485274846542,
+      "grad_norm": 0.2753259241580963,
+      "learning_rate": 1.238846737481032e-05,
+      "loss": 1.7193,
+      "mean_token_accuracy": 0.6123683395981788,
+      "num_tokens": 598467233.0,
+      "step": 13800
+    },
+    {
+      "epoch": 1.931236637009449,
+      "grad_norm": 0.2765011489391327,
+      "learning_rate": 1.2278107325148296e-05,
+      "loss": 1.7185,
+      "mean_token_accuracy": 0.6119060663878918,
+      "num_tokens": 611505054.0,
+      "step": 14000
+    },
+    {
+      "epoch": 1.9588247465342437,
+      "grad_norm": 0.29317325353622437,
+      "learning_rate": 1.2167747275486274e-05,
+      "loss": 1.7164,
+      "mean_token_accuracy": 0.6127070464193821,
+      "num_tokens": 624515618.0,
+      "step": 14200
+    },
+    {
+      "epoch": 1.9864128560590384,
+      "grad_norm": 0.27798038721084595,
+      "learning_rate": 1.2057387225824253e-05,
+      "loss": 1.7148,
+      "mean_token_accuracy": 0.6131316345930099,
+      "num_tokens": 637507621.0,
+      "step": 14400
+    },
+    {
+      "epoch": 2.0139319953100214,
+      "grad_norm": 0.28348833322525024,
+      "learning_rate": 1.194702717616223e-05,
+      "loss": 1.7077,
+      "mean_token_accuracy": 0.6135270874900627,
+      "num_tokens": 650501333.0,
+      "step": 14600
+    },
+    {
+      "epoch": 2.041520104834816,
+      "grad_norm": 0.2825703024864197,
+      "learning_rate": 1.1836667126500208e-05,
+      "loss": 1.7,
+      "mean_token_accuracy": 0.6154656463861465,
+      "num_tokens": 663491183.0,
+      "step": 14800
+    },
+    {
+      "epoch": 2.069108214359611,
+      "grad_norm": 0.2799076437950134,
+      "learning_rate": 1.1726307076838186e-05,
+      "loss": 1.6941,
+      "mean_token_accuracy": 0.6160167968273162,
+      "num_tokens": 676489125.0,
+      "step": 15000
+    },
+    {
+      "epoch": 2.0966963238844056,
+      "grad_norm": 0.2782443165779114,
+      "learning_rate": 1.1615947027176161e-05,
+      "loss": 1.709,
+      "mean_token_accuracy": 0.6133450974524021,
+      "num_tokens": 689511656.0,
+      "step": 15200
+    },
+    {
+      "epoch": 2.124284433409201,
+      "grad_norm": 0.2791282832622528,
+      "learning_rate": 1.150558697751414e-05,
+      "loss": 1.7012,
+      "mean_token_accuracy": 0.6146386332809926,
+      "num_tokens": 702532680.0,
+      "step": 15400
+    },
+    {
+      "epoch": 2.1518725429339955,
+      "grad_norm": 0.2790575921535492,
+      "learning_rate": 1.1395226927852118e-05,
+      "loss": 1.7058,
+      "mean_token_accuracy": 0.6137663789093495,
+      "num_tokens": 715555115.0,
+      "step": 15600
+    },
+    {
+      "epoch": 2.1794606524587903,
+      "grad_norm": 0.2901487648487091,
+      "learning_rate": 1.1284866878190097e-05,
+      "loss": 1.6948,
+      "mean_token_accuracy": 0.6157194478809833,
+      "num_tokens": 728582765.0,
+      "step": 15800
+    },
+    {
+      "epoch": 2.207048761983585,
+      "grad_norm": 0.2835938334465027,
+      "learning_rate": 1.1174506828528073e-05,
+      "loss": 1.698,
+      "mean_token_accuracy": 0.615332849919796,
+      "num_tokens": 741592804.0,
+      "step": 16000
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 0.28976014256477356,
+      "learning_rate": 1.1064146778866052e-05,
+      "loss": 1.7036,
+      "mean_token_accuracy": 0.6142940573394299,
+      "num_tokens": 754612801.0,
+      "step": 16200
+    },
+    {
+      "epoch": 2.2622249810331745,
+      "grad_norm": 0.28366798162460327,
+      "learning_rate": 1.095378672920403e-05,
+      "loss": 1.6955,
+      "mean_token_accuracy": 0.6157566201686859,
+      "num_tokens": 767616919.0,
+      "step": 16400
+    },
+    {
+      "epoch": 2.2898130905579697,
+      "grad_norm": 0.28289395570755005,
+      "learning_rate": 1.0843426679542006e-05,
+      "loss": 1.6938,
+      "mean_token_accuracy": 0.6161167666316032,
+      "num_tokens": 780634732.0,
+      "step": 16600
+    },
+    {
+      "epoch": 2.3174012000827644,
+      "grad_norm": 0.2955986559391022,
+      "learning_rate": 1.0733066629879984e-05,
+      "loss": 1.6947,
+      "mean_token_accuracy": 0.6157604093849659,
+      "num_tokens": 793645866.0,
+      "step": 16800
+    },
+    {
+      "epoch": 2.344989309607559,
+      "grad_norm": 0.2817966341972351,
+      "learning_rate": 1.0622706580217962e-05,
+      "loss": 1.7028,
+      "mean_token_accuracy": 0.6145315931737423,
+      "num_tokens": 806653207.0,
+      "step": 17000
+    },
+    {
+      "epoch": 2.372577419132354,
+      "grad_norm": 0.28311964869499207,
+      "learning_rate": 1.051234653055594e-05,
+      "loss": 1.696,
+      "mean_token_accuracy": 0.615741535872221,
+      "num_tokens": 819673655.0,
+      "step": 17200
+    },
+    {
+      "epoch": 2.4001655286571486,
+      "grad_norm": 0.28408530354499817,
+      "learning_rate": 1.0401986480893918e-05,
+      "loss": 1.7001,
+      "mean_token_accuracy": 0.6146831949055195,
+      "num_tokens": 832689455.0,
+      "step": 17400
+    },
+    {
+      "epoch": 2.427753638181944,
+      "grad_norm": 0.29735735058784485,
+      "learning_rate": 1.0291626431231896e-05,
+      "loss": 1.6871,
+      "mean_token_accuracy": 0.6170836135745048,
+      "num_tokens": 845698323.0,
+      "step": 17600
+    },
+    {
+      "epoch": 2.4553417477067385,
+      "grad_norm": 0.3009927272796631,
+      "learning_rate": 1.0181266381569871e-05,
+      "loss": 1.6949,
+      "mean_token_accuracy": 0.6159922151267528,
+      "num_tokens": 858702827.0,
+      "step": 17800
+    },
+    {
+      "epoch": 2.4829298572315333,
+      "grad_norm": 0.29742059111595154,
+      "learning_rate": 1.007090633190785e-05,
+      "loss": 1.6968,
+      "mean_token_accuracy": 0.6157895983755588,
+      "num_tokens": 871701677.0,
+      "step": 18000
+    },
+    {
+      "epoch": 2.510517966756328,
+      "grad_norm": 0.2822323143482208,
+      "learning_rate": 9.960546282245828e-06,
+      "loss": 1.7002,
+      "mean_token_accuracy": 0.6144832235574722,
+      "num_tokens": 884719034.0,
+      "step": 18200
+    },
+    {
+      "epoch": 2.5381060762811227,
+      "grad_norm": 0.29363512992858887,
+      "learning_rate": 9.850186232583807e-06,
+      "loss": 1.696,
+      "mean_token_accuracy": 0.6158060549199581,
+      "num_tokens": 897716370.0,
+      "step": 18400
+    },
+    {
+      "epoch": 2.565694185805918,
+      "grad_norm": 0.29309144616127014,
+      "learning_rate": 9.739826182921783e-06,
+      "loss": 1.694,
+      "mean_token_accuracy": 0.6163165862858295,
+      "num_tokens": 910725195.0,
+      "step": 18600
+    },
+    {
+      "epoch": 2.593282295330712,
+      "grad_norm": 0.2888060510158539,
+      "learning_rate": 9.62946613325976e-06,
+      "loss": 1.6957,
+      "mean_token_accuracy": 0.6159252269566059,
+      "num_tokens": 923732160.0,
+      "step": 18800
+    },
+    {
+      "epoch": 2.6208704048555074,
+      "grad_norm": 0.28722092509269714,
+      "learning_rate": 9.519106083597739e-06,
+      "loss": 1.7,
+      "mean_token_accuracy": 0.6145291210711002,
+      "num_tokens": 936739185.0,
+      "step": 19000
+    },
+    {
+      "epoch": 2.648458514380302,
+      "grad_norm": 0.29655253887176514,
+      "learning_rate": 9.408746033935715e-06,
+      "loss": 1.6954,
+      "mean_token_accuracy": 0.6159876026213169,
+      "num_tokens": 949725573.0,
+      "step": 19200
+    },
+    {
+      "epoch": 2.676046623905097,
+      "grad_norm": 0.29758745431900024,
+      "learning_rate": 9.298385984273694e-06,
+      "loss": 1.6958,
+      "mean_token_accuracy": 0.616194871366024,
+      "num_tokens": 962696837.0,
+      "step": 19400
+    },
+    {
+      "epoch": 2.7036347334298916,
+      "grad_norm": 0.29457658529281616,
+      "learning_rate": 9.188025934611672e-06,
+      "loss": 1.692,
+      "mean_token_accuracy": 0.6167295287549496,
+      "num_tokens": 975699859.0,
+      "step": 19600
+    },
+    {
+      "epoch": 2.7312228429546863,
+      "grad_norm": 0.29310283064842224,
+      "learning_rate": 9.077665884949649e-06,
+      "loss": 1.6948,
+      "mean_token_accuracy": 0.6162155202031135,
+      "num_tokens": 988705637.0,
+      "step": 19800
+    },
+    {
+      "epoch": 2.7588109524794815,
+      "grad_norm": 0.2922320067882538,
+      "learning_rate": 8.967305835287626e-06,
+      "loss": 1.6901,
+      "mean_token_accuracy": 0.6163058173656464,
+      "num_tokens": 1001754945.0,
+      "step": 20000
+    },
+    {
+      "epoch": 2.7863990620042762,
+      "grad_norm": 0.2901257276535034,
+      "learning_rate": 8.856945785625604e-06,
+      "loss": 1.6951,
+      "mean_token_accuracy": 0.6158974586427212,
+      "num_tokens": 1014746336.0,
+      "step": 20200
+    },
+    {
+      "epoch": 2.813987171529071,
+      "grad_norm": 0.28679758310317993,
+      "learning_rate": 8.746585735963581e-06,
+      "loss": 1.6858,
+      "mean_token_accuracy": 0.6173767544329166,
+      "num_tokens": 1027763913.0,
+      "step": 20400
+    },
+    {
+      "epoch": 2.8415752810538657,
+      "grad_norm": 0.2966858744621277,
+      "learning_rate": 8.63622568630156e-06,
+      "loss": 1.686,
+      "mean_token_accuracy": 0.6175419771671296,
+      "num_tokens": 1040766281.0,
+      "step": 20600
+    },
+    {
+      "epoch": 2.8691633905786604,
+      "grad_norm": 0.300108939409256,
+      "learning_rate": 8.525865636639538e-06,
+      "loss": 1.694,
+      "mean_token_accuracy": 0.6159957525134087,
+      "num_tokens": 1053798269.0,
+      "step": 20800
+    },
+    {
+      "epoch": 2.8967515001034556,
+      "grad_norm": 0.2970072031021118,
+      "learning_rate": 8.415505586977515e-06,
+      "loss": 1.6881,
+      "mean_token_accuracy": 0.6167993746697903,
+      "num_tokens": 1066815504.0,
+      "step": 21000
     }
   ],
   "logging_steps": 200,
@@ -234,7 +972,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 5.41112571879909e+18,
+  "total_flos": 2.469866946100514e+19,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null