{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 637,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0015698587127158557,
      "grad_norm": 0.982062816619873,
      "learning_rate": 7.8125e-08,
      "logits/chosen": -1.9796161651611328,
      "logits/rejected": -1.7990202903747559,
      "logps/chosen": -276.0213317871094,
      "logps/rejected": -220.55433654785156,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.015698587127158554,
      "grad_norm": 0.9399773478507996,
      "learning_rate": 7.8125e-07,
      "logits/chosen": -2.2138850688934326,
      "logits/rejected": -2.1262121200561523,
      "logps/chosen": -290.8321533203125,
      "logps/rejected": -228.361328125,
      "loss": 0.6929,
      "rewards/accuracies": 0.4652777910232544,
      "rewards/chosen": 0.00492230337113142,
      "rewards/margins": 0.0006595260347239673,
      "rewards/rejected": 0.0042627775110304356,
      "step": 10
    },
    {
      "epoch": 0.03139717425431711,
      "grad_norm": 1.019684076309204,
      "learning_rate": 1.5625e-06,
      "logits/chosen": -2.3072190284729004,
      "logits/rejected": -2.201758861541748,
      "logps/chosen": -265.79144287109375,
      "logps/rejected": -252.1865997314453,
      "loss": 0.6895,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.029890870675444603,
      "rewards/margins": 0.009029369801282883,
      "rewards/rejected": 0.02086150087416172,
      "step": 20
    },
    {
      "epoch": 0.04709576138147567,
      "grad_norm": 0.9330567121505737,
      "learning_rate": 2.3437500000000002e-06,
      "logits/chosen": -2.312687635421753,
      "logits/rejected": -2.206019878387451,
      "logps/chosen": -268.0202331542969,
      "logps/rejected": -250.2025146484375,
      "loss": 0.6844,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.05118671804666519,
      "rewards/margins": 0.019565049558877945,
      "rewards/rejected": 0.03162166848778725,
      "step": 30
    },
    {
      "epoch": 0.06279434850863422,
      "grad_norm": 0.9986972808837891,
      "learning_rate": 3.125e-06,
      "logits/chosen": -2.2849414348602295,
      "logits/rejected": -2.1605541706085205,
      "logps/chosen": -248.8426971435547,
      "logps/rejected": -222.5389404296875,
      "loss": 0.675,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.06607525795698166,
      "rewards/margins": 0.039594851434230804,
      "rewards/rejected": 0.026480406522750854,
      "step": 40
    },
    {
      "epoch": 0.07849293563579278,
      "grad_norm": 1.18948495388031,
      "learning_rate": 3.90625e-06,
      "logits/chosen": -2.237248659133911,
      "logits/rejected": -2.190307855606079,
      "logps/chosen": -268.77899169921875,
      "logps/rejected": -278.4407958984375,
      "loss": 0.6605,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.00511029502376914,
      "rewards/margins": 0.05497710779309273,
      "rewards/rejected": -0.04986681044101715,
      "step": 50
    },
    {
      "epoch": 0.09419152276295134,
      "grad_norm": 1.285398006439209,
      "learning_rate": 4.6875000000000004e-06,
      "logits/chosen": -2.265840768814087,
      "logits/rejected": -2.1778528690338135,
      "logps/chosen": -272.609130859375,
      "logps/rejected": -277.2625732421875,
      "loss": 0.6429,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07459741830825806,
      "rewards/margins": 0.12982353568077087,
      "rewards/rejected": -0.20442095398902893,
      "step": 60
    },
    {
      "epoch": 0.10989010989010989,
      "grad_norm": 1.8551888465881348,
      "learning_rate": 4.998647417232375e-06,
      "logits/chosen": -2.1861605644226074,
      "logits/rejected": -2.0909242630004883,
      "logps/chosen": -267.07611083984375,
      "logps/rejected": -259.22210693359375,
      "loss": 0.6386,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.2929137647151947,
      "rewards/margins": 0.1055971160531044,
      "rewards/rejected": -0.3985108435153961,
      "step": 70
    },
    {
      "epoch": 0.12558869701726844,
      "grad_norm": 1.9893733263015747,
      "learning_rate": 4.990386933279973e-06,
      "logits/chosen": -2.184288740158081,
      "logits/rejected": -2.1073215007781982,
      "logps/chosen": -290.60943603515625,
      "logps/rejected": -284.17425537109375,
      "loss": 0.6057,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.30957552790641785,
      "rewards/margins": 0.2696777582168579,
      "rewards/rejected": -0.5792533159255981,
      "step": 80
    },
    {
      "epoch": 0.141287284144427,
      "grad_norm": 3.498678684234619,
      "learning_rate": 4.974642195009681e-06,
      "logits/chosen": -2.1730189323425293,
      "logits/rejected": -2.092939853668213,
      "logps/chosen": -326.27838134765625,
      "logps/rejected": -310.67767333984375,
      "loss": 0.6017,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5229882001876831,
      "rewards/margins": 0.20935317873954773,
      "rewards/rejected": -0.7323412895202637,
      "step": 90
    },
    {
      "epoch": 0.15698587127158556,
      "grad_norm": 2.8239424228668213,
      "learning_rate": 4.951460519416228e-06,
      "logits/chosen": -2.254025936126709,
      "logits/rejected": -2.1698710918426514,
      "logps/chosen": -328.6602783203125,
      "logps/rejected": -333.0082092285156,
      "loss": 0.6099,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4128250479698181,
      "rewards/margins": 0.2569137513637543,
      "rewards/rejected": -0.6697388887405396,
      "step": 100
    },
    {
      "epoch": 0.15698587127158556,
      "eval_logits/chosen": -2.2190940380096436,
      "eval_logits/rejected": -2.130751371383667,
      "eval_logps/chosen": -304.2706604003906,
      "eval_logps/rejected": -307.0733642578125,
      "eval_loss": 0.59989333152771,
      "eval_rewards/accuracies": 0.636904776096344,
      "eval_rewards/chosen": -0.45707327127456665,
      "eval_rewards/margins": 0.2184356451034546,
      "eval_rewards/rejected": -0.675508975982666,
      "eval_runtime": 219.8057,
      "eval_samples_per_second": 9.099,
      "eval_steps_per_second": 0.382,
      "step": 100
    },
    {
      "epoch": 0.1726844583987441,
      "grad_norm": 4.492749214172363,
      "learning_rate": 4.920911573406925e-06,
      "logits/chosen": -2.2060790061950684,
      "logits/rejected": -2.0495288372039795,
      "logps/chosen": -324.7986755371094,
      "logps/rejected": -339.40802001953125,
      "loss": 0.5822,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5525876879692078,
      "rewards/margins": 0.44238606095314026,
      "rewards/rejected": -0.9949736595153809,
      "step": 110
    },
    {
      "epoch": 0.18838304552590268,
      "grad_norm": 4.563241481781006,
      "learning_rate": 4.883087164434672e-06,
      "logits/chosen": -2.0682525634765625,
      "logits/rejected": -2.0306172370910645,
      "logps/chosen": -367.96966552734375,
      "logps/rejected": -406.1026611328125,
      "loss": 0.5553,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.9666538238525391,
      "rewards/margins": 0.4744674563407898,
      "rewards/rejected": -1.4411213397979736,
      "step": 120
    },
    {
      "epoch": 0.20408163265306123,
      "grad_norm": 4.36543607711792,
      "learning_rate": 4.8381009645929044e-06,
      "logits/chosen": -1.958156943321228,
      "logits/rejected": -1.8605877161026,
      "logps/chosen": -372.92083740234375,
      "logps/rejected": -398.33209228515625,
      "loss": 0.5609,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.9711011052131653,
      "rewards/margins": 0.45527857542037964,
      "rewards/rejected": -1.426379680633545,
      "step": 130
    },
    {
      "epoch": 0.21978021978021978,
      "grad_norm": 3.9676990509033203,
      "learning_rate": 4.786088169001671e-06,
      "logits/chosen": -1.9477580785751343,
      "logits/rejected": -1.9076995849609375,
      "logps/chosen": -364.2967224121094,
      "logps/rejected": -398.6386413574219,
      "loss": 0.5805,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.2606209516525269,
      "rewards/margins": 0.37748971581459045,
      "rewards/rejected": -1.6381107568740845,
      "step": 140
    },
    {
      "epoch": 0.23547880690737832,
      "grad_norm": 4.215748310089111,
      "learning_rate": 4.727205089511466e-06,
      "logits/chosen": -1.8670552968978882,
      "logits/rejected": -1.8230712413787842,
      "logps/chosen": -380.49237060546875,
      "logps/rejected": -442.67596435546875,
      "loss": 0.5594,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.3134872913360596,
      "rewards/margins": 0.47393256425857544,
      "rewards/rejected": -1.7874199151992798,
      "step": 150
    },
    {
      "epoch": 0.25117739403453687,
      "grad_norm": 4.537078380584717,
      "learning_rate": 4.661628684945851e-06,
      "logits/chosen": -1.8435173034667969,
      "logits/rejected": -1.7608001232147217,
      "logps/chosen": -380.8707580566406,
      "logps/rejected": -421.7408142089844,
      "loss": 0.5537,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.0851117372512817,
      "rewards/margins": 0.5639781355857849,
      "rewards/rejected": -1.6490898132324219,
      "step": 160
    },
    {
      "epoch": 0.2668759811616955,
      "grad_norm": 4.127157211303711,
      "learning_rate": 4.5895560292946e-06,
      "logits/chosen": -1.8356807231903076,
      "logits/rejected": -1.7069345712661743,
      "logps/chosen": -406.5726623535156,
      "logps/rejected": -423.89862060546875,
      "loss": 0.5395,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.350484848022461,
      "rewards/margins": 0.5463389754295349,
      "rewards/rejected": -1.8968238830566406,
      "step": 170
    },
    {
      "epoch": 0.282574568288854,
      "grad_norm": 4.295073986053467,
      "learning_rate": 4.511203719455588e-06,
      "logits/chosen": -1.805424690246582,
      "logits/rejected": -1.675389051437378,
      "logps/chosen": -404.70721435546875,
      "logps/rejected": -431.9833984375,
      "loss": 0.539,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.289776086807251,
      "rewards/margins": 0.7749110460281372,
      "rewards/rejected": -2.0646872520446777,
      "step": 180
    },
    {
      "epoch": 0.29827315541601257,
      "grad_norm": 4.170295715332031,
      "learning_rate": 4.426807224305315e-06,
      "logits/chosen": -1.7862762212753296,
      "logits/rejected": -1.635716438293457,
      "logps/chosen": -372.5980529785156,
      "logps/rejected": -416.7577209472656,
      "loss": 0.5637,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.134557843208313,
      "rewards/margins": 0.753765344619751,
      "rewards/rejected": -1.888323187828064,
      "step": 190
    },
    {
      "epoch": 0.3139717425431711,
      "grad_norm": 4.384995937347412,
      "learning_rate": 4.336620177054269e-06,
      "logits/chosen": -1.7506341934204102,
      "logits/rejected": -1.6495962142944336,
      "logps/chosen": -347.5675048828125,
      "logps/rejected": -391.43731689453125,
      "loss": 0.5468,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.009634256362915,
      "rewards/margins": 0.6290403604507446,
      "rewards/rejected": -1.6386747360229492,
      "step": 200
    },
    {
      "epoch": 0.3139717425431711,
      "eval_logits/chosen": -1.607847809791565,
      "eval_logits/rejected": -1.5156790018081665,
      "eval_logps/chosen": -379.1949768066406,
      "eval_logps/rejected": -409.77398681640625,
      "eval_loss": 0.5353844165802002,
      "eval_rewards/accuracies": 0.6815476417541504,
      "eval_rewards/chosen": -1.2063161134719849,
      "eval_rewards/margins": 0.49619969725608826,
      "eval_rewards/rejected": -1.702515721321106,
      "eval_runtime": 219.5785,
      "eval_samples_per_second": 9.108,
      "eval_steps_per_second": 0.383,
      "step": 200
    },
    {
      "epoch": 0.32967032967032966,
      "grad_norm": 4.753127098083496,
      "learning_rate": 4.240913613013785e-06,
      "logits/chosen": -1.6041982173919678,
      "logits/rejected": -1.4586137533187866,
      "logps/chosen": -403.050537109375,
      "logps/rejected": -434.6443786621094,
      "loss": 0.5589,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.2178317308425903,
      "rewards/margins": 0.7227036952972412,
      "rewards/rejected": -1.940535306930542,
      "step": 210
    },
    {
      "epoch": 0.3453689167974882,
      "grad_norm": 4.243059158325195,
      "learning_rate": 4.139975155065109e-06,
      "logits/chosen": -1.545710802078247,
      "logits/rejected": -1.4544004201889038,
      "logps/chosen": -359.60174560546875,
      "logps/rejected": -431.0966796875,
      "loss": 0.5293,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.2607206106185913,
      "rewards/margins": 0.6980492472648621,
      "rewards/rejected": -1.9587697982788086,
      "step": 220
    },
    {
      "epoch": 0.36106750392464676,
      "grad_norm": 4.250452995300293,
      "learning_rate": 4.034108149278544e-06,
      "logits/chosen": -1.5040862560272217,
      "logits/rejected": -1.419395923614502,
      "logps/chosen": -438.34979248046875,
      "logps/rejected": -493.54608154296875,
      "loss": 0.5378,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.6042639017105103,
      "rewards/margins": 0.6156054735183716,
      "rewards/rejected": -2.2198691368103027,
      "step": 230
    },
    {
      "epoch": 0.37676609105180536,
      "grad_norm": 5.608371734619141,
      "learning_rate": 3.923630753280358e-06,
      "logits/chosen": -1.5527160167694092,
      "logits/rejected": -1.4736815690994263,
      "logps/chosen": -357.8080749511719,
      "logps/rejected": -439.68218994140625,
      "loss": 0.4945,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9829418063163757,
      "rewards/margins": 0.868217945098877,
      "rewards/rejected": -1.8511598110198975,
      "step": 240
    },
    {
      "epoch": 0.3924646781789639,
      "grad_norm": 4.367254734039307,
      "learning_rate": 3.80887498010715e-06,
      "logits/chosen": -1.4042975902557373,
      "logits/rejected": -1.283562421798706,
      "logps/chosen": -437.2142639160156,
      "logps/rejected": -488.03826904296875,
      "loss": 0.5232,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.8854888677597046,
      "rewards/margins": 0.7756179571151733,
      "rewards/rejected": -2.661106824874878,
      "step": 250
    },
    {
      "epoch": 0.40816326530612246,
      "grad_norm": 3.6756815910339355,
      "learning_rate": 3.690185700421145e-06,
      "logits/chosen": -1.534239411354065,
      "logits/rejected": -1.4822893142700195,
      "logps/chosen": -338.0312805175781,
      "logps/rejected": -372.50274658203125,
      "loss": 0.5643,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9809848666191101,
      "rewards/margins": 0.43755441904067993,
      "rewards/rejected": -1.41853928565979,
      "step": 260
    },
    {
      "epoch": 0.423861852433281,
      "grad_norm": 4.209615707397461,
      "learning_rate": 3.567919606085004e-06,
      "logits/chosen": -1.6105585098266602,
      "logits/rejected": -1.5416157245635986,
      "logps/chosen": -353.83245849609375,
      "logps/rejected": -427.657470703125,
      "loss": 0.5363,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.8973706364631653,
      "rewards/margins": 0.6859413385391235,
      "rewards/rejected": -1.5833120346069336,
      "step": 270
    },
    {
      "epoch": 0.43956043956043955,
      "grad_norm": 5.80835485458374,
      "learning_rate": 3.442444138210883e-06,
      "logits/chosen": -1.3900154829025269,
      "logits/rejected": -1.286752700805664,
      "logps/chosen": -383.26959228515625,
      "logps/rejected": -396.59735107421875,
      "loss": 0.5434,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.367221474647522,
      "rewards/margins": 0.605404257774353,
      "rewards/rejected": -1.972625732421875,
      "step": 280
    },
    {
      "epoch": 0.4552590266875981,
      "grad_norm": 5.807376861572266,
      "learning_rate": 3.314136382905234e-06,
      "logits/chosen": -1.4290958642959595,
      "logits/rejected": -1.3117034435272217,
      "logps/chosen": -405.759765625,
      "logps/rejected": -466.0645446777344,
      "loss": 0.5277,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.4295778274536133,
      "rewards/margins": 0.7734361886978149,
      "rewards/rejected": -2.2030138969421387,
      "step": 290
    },
    {
      "epoch": 0.47095761381475665,
      "grad_norm": 4.498719215393066,
      "learning_rate": 3.1833819380279028e-06,
      "logits/chosen": -1.4070186614990234,
      "logits/rejected": -1.3326956033706665,
      "logps/chosen": -398.8482666015625,
      "logps/rejected": -497.5279235839844,
      "loss": 0.5195,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.5320394039154053,
      "rewards/margins": 0.9529226422309875,
      "rewards/rejected": -2.484961986541748,
      "step": 300
    },
    {
      "epoch": 0.47095761381475665,
      "eval_logits/chosen": -1.452673077583313,
      "eval_logits/rejected": -1.3522714376449585,
      "eval_logps/chosen": -418.378173828125,
      "eval_logps/rejected": -467.829345703125,
      "eval_loss": 0.5227062702178955,
      "eval_rewards/accuracies": 0.7083333134651184,
      "eval_rewards/chosen": -1.5981483459472656,
      "eval_rewards/margins": 0.684920608997345,
      "eval_rewards/rejected": -2.2830684185028076,
      "eval_runtime": 219.6897,
      "eval_samples_per_second": 9.104,
      "eval_steps_per_second": 0.382,
      "step": 300
    },
    {
      "epoch": 0.48665620094191525,
      "grad_norm": 6.346375465393066,
      "learning_rate": 3.050573754371228e-06,
      "logits/chosen": -1.4891157150268555,
      "logits/rejected": -1.425148844718933,
      "logps/chosen": -399.7137451171875,
      "logps/rejected": -429.02960205078125,
      "loss": 0.5355,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.2991411685943604,
      "rewards/margins": 0.6303215026855469,
      "rewards/rejected": -1.9294626712799072,
      "step": 310
    },
    {
      "epoch": 0.5023547880690737,
      "grad_norm": 5.018222808837891,
      "learning_rate": 2.916110954741667e-06,
      "logits/chosen": -1.4688920974731445,
      "logits/rejected": -1.295971155166626,
      "logps/chosen": -429.50250244140625,
      "logps/rejected": -448.509521484375,
      "loss": 0.4945,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.343754529953003,
      "rewards/margins": 0.7055485844612122,
      "rewards/rejected": -2.0493030548095703,
      "step": 320
    },
    {
      "epoch": 0.5180533751962323,
      "grad_norm": 5.533540725708008,
      "learning_rate": 2.7803976344929497e-06,
      "logits/chosen": -1.3964585065841675,
      "logits/rejected": -1.2397369146347046,
      "logps/chosen": -450.6763610839844,
      "logps/rejected": -513.90234375,
      "loss": 0.5211,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.7644832134246826,
      "rewards/margins": 0.8988251686096191,
      "rewards/rejected": -2.663308620452881,
      "step": 330
    },
    {
      "epoch": 0.533751962323391,
      "grad_norm": 4.254801273345947,
      "learning_rate": 2.6438416471154277e-06,
      "logits/chosen": -1.420155644416809,
      "logits/rejected": -1.355759859085083,
      "logps/chosen": -393.2152099609375,
      "logps/rejected": -455.6507263183594,
      "loss": 0.5032,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.4955414533615112,
      "rewards/margins": 0.7609378099441528,
      "rewards/rejected": -2.256479501724243,
      "step": 340
    },
    {
      "epoch": 0.5494505494505495,
      "grad_norm": 5.497036933898926,
      "learning_rate": 2.5068533785312673e-06,
      "logits/chosen": -1.2714860439300537,
      "logits/rejected": -1.206830382347107,
      "logps/chosen": -424.79864501953125,
      "logps/rejected": -484.2491760253906,
      "loss": 0.5265,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.5535635948181152,
      "rewards/margins": 0.7852663993835449,
      "rewards/rejected": -2.33882999420166,
      "step": 350
    },
    {
      "epoch": 0.565149136577708,
      "grad_norm": 4.5596208572387695,
      "learning_rate": 2.369844513779026e-06,
      "logits/chosen": -1.2018563747406006,
      "logits/rejected": -1.115635871887207,
      "logps/chosen": -440.96942138671875,
      "logps/rejected": -488.6031799316406,
      "loss": 0.5316,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.7989181280136108,
      "rewards/margins": 0.8467502593994141,
      "rewards/rejected": -2.6456682682037354,
      "step": 360
    },
    {
      "epoch": 0.5808477237048666,
      "grad_norm": 4.705105781555176,
      "learning_rate": 2.2332267997940514e-06,
      "logits/chosen": -1.3089182376861572,
      "logits/rejected": -1.1716320514678955,
      "logps/chosen": -400.7124938964844,
      "logps/rejected": -441.3446350097656,
      "loss": 0.5463,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4626152515411377,
      "rewards/margins": 0.795612096786499,
      "rewards/rejected": -2.2582273483276367,
      "step": 370
    },
    {
      "epoch": 0.5965463108320251,
      "grad_norm": 4.903963565826416,
      "learning_rate": 2.097410808002869e-06,
      "logits/chosen": -1.3345696926116943,
      "logits/rejected": -1.196610927581787,
      "logps/chosen": -416.953125,
      "logps/rejected": -442.70611572265625,
      "loss": 0.5065,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.265023946762085,
      "rewards/margins": 0.7792559862136841,
      "rewards/rejected": -2.0442800521850586,
      "step": 380
    },
    {
      "epoch": 0.6122448979591837,
      "grad_norm": 5.188351631164551,
      "learning_rate": 1.962804700450265e-06,
      "logits/chosen": -1.2588812112808228,
      "logits/rejected": -1.1723003387451172,
      "logps/chosen": -375.08428955078125,
      "logps/rejected": -457.165771484375,
      "loss": 0.5228,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.3058245182037354,
      "rewards/margins": 0.9940802454948425,
      "rewards/rejected": -2.2999048233032227,
      "step": 390
    },
    {
      "epoch": 0.6279434850863422,
      "grad_norm": 6.60071325302124,
      "learning_rate": 1.8298130031671974e-06,
      "logits/chosen": -1.3610254526138306,
      "logits/rejected": -1.1776959896087646,
      "logps/chosen": -445.6390686035156,
      "logps/rejected": -512.9346923828125,
      "loss": 0.4895,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.6864149570465088,
      "rewards/margins": 0.9683998823165894,
      "rewards/rejected": -2.6548147201538086,
      "step": 400
    },
    {
      "epoch": 0.6279434850863422,
      "eval_logits/chosen": -1.2180150747299194,
      "eval_logits/rejected": -1.1070232391357422,
      "eval_logps/chosen": -444.1170959472656,
      "eval_logps/rejected": -506.0622253417969,
      "eval_loss": 0.514235258102417,
      "eval_rewards/accuracies": 0.699404776096344,
      "eval_rewards/chosen": -1.8555371761322021,
      "eval_rewards/margins": 0.809860348701477,
      "eval_rewards/rejected": -2.6653974056243896,
      "eval_runtime": 220.0748,
      "eval_samples_per_second": 9.088,
      "eval_steps_per_second": 0.382,
      "step": 400
    },
    {
      "epoch": 0.6436420722135008,
      "grad_norm": 3.98221755027771,
      "learning_rate": 1.6988353904658495e-06,
      "logits/chosen": -1.2132154703140259,
      "logits/rejected": -1.1685426235198975,
      "logps/chosen": -435.8744201660156,
      "logps/rejected": -512.9607543945312,
      "loss": 0.5104,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.587599515914917,
      "rewards/margins": 1.0233564376831055,
      "rewards/rejected": -2.6109557151794434,
      "step": 410
    },
    {
      "epoch": 0.6593406593406593,
      "grad_norm": 6.315725803375244,
      "learning_rate": 1.5702654838153641e-06,
      "logits/chosen": -1.1728157997131348,
      "logits/rejected": -1.1026521921157837,
      "logps/chosen": -421.05963134765625,
      "logps/rejected": -512.4940185546875,
      "loss": 0.4971,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.7201759815216064,
      "rewards/margins": 0.8986526727676392,
      "rewards/rejected": -2.618828535079956,
      "step": 420
    },
    {
      "epoch": 0.6750392464678179,
      "grad_norm": 5.234571933746338,
      "learning_rate": 1.4444896689079142e-06,
      "logits/chosen": -1.1644630432128906,
      "logits/rejected": -1.0723572969436646,
      "logps/chosen": -465.34661865234375,
      "logps/rejected": -545.7144165039062,
      "loss": 0.5029,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.9419937133789062,
      "rewards/margins": 1.0666325092315674,
      "rewards/rejected": -3.0086259841918945,
      "step": 430
    },
    {
      "epoch": 0.6907378335949764,
      "grad_norm": 5.252328395843506,
      "learning_rate": 1.3218859344701634e-06,
      "logits/chosen": -1.1691018342971802,
      "logits/rejected": -1.120827317237854,
      "logps/chosen": -424.86639404296875,
      "logps/rejected": -522.9156494140625,
      "loss": 0.4923,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.887139081954956,
      "rewards/margins": 1.0102875232696533,
      "rewards/rejected": -2.8974266052246094,
      "step": 440
    },
    {
      "epoch": 0.706436420722135,
      "grad_norm": 7.3930206298828125,
      "learning_rate": 1.2028227363097583e-06,
      "logits/chosen": -1.1967122554779053,
      "logits/rejected": -1.157003402709961,
      "logps/chosen": -483.7662048339844,
      "logps/rejected": -568.5826416015625,
      "loss": 0.4961,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.155179023742676,
      "rewards/margins": 0.9878844022750854,
      "rewards/rejected": -3.14306378364563,
      "step": 450
    },
    {
      "epoch": 0.7221350078492935,
      "grad_norm": 3.9247488975524902,
      "learning_rate": 1.0876578900107053e-06,
      "logits/chosen": -1.2375166416168213,
      "logits/rejected": -1.1339408159255981,
      "logps/chosen": -465.45458984375,
      "logps/rejected": -551.1287231445312,
      "loss": 0.4842,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.9650770425796509,
      "rewards/margins": 1.0327082872390747,
      "rewards/rejected": -2.9977853298187256,
      "step": 460
    },
    {
      "epoch": 0.7378335949764521,
      "grad_norm": 7.045323848724365,
      "learning_rate": 9.767374956053584e-07,
      "logits/chosen": -1.1633169651031494,
      "logits/rejected": -1.0927058458328247,
      "logps/chosen": -469.948486328125,
      "logps/rejected": -547.9520874023438,
      "loss": 0.4992,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.16100811958313,
      "rewards/margins": 0.9476286172866821,
      "rewards/rejected": -3.1086366176605225,
      "step": 470
    },
    {
      "epoch": 0.7535321821036107,
      "grad_norm": 5.355714797973633,
      "learning_rate": 8.703948974546592e-07,
      "logits/chosen": -1.2294596433639526,
      "logits/rejected": -1.1127874851226807,
      "logps/chosen": -500.1397399902344,
      "logps/rejected": -558.7158813476562,
      "loss": 0.508,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.36497163772583,
      "rewards/margins": 0.9023281335830688,
      "rewards/rejected": -3.2672996520996094,
      "step": 480
    },
    {
      "epoch": 0.7692307692307693,
      "grad_norm": 5.423379421234131,
      "learning_rate": 7.689496824624526e-07,
      "logits/chosen": -1.206969976425171,
      "logits/rejected": -1.0124098062515259,
      "logps/chosen": -503.8736267089844,
      "logps/rejected": -560.7037353515625,
      "loss": 0.5118,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.366530656814575,
      "rewards/margins": 1.0277622938156128,
      "rewards/rejected": -3.3942928314208984,
      "step": 490
    },
    {
      "epoch": 0.7849293563579278,
      "grad_norm": 4.5977630615234375,
      "learning_rate": 6.7270671963451e-07,
      "logits/chosen": -1.24196195602417,
      "logits/rejected": -1.166027307510376,
      "logps/chosen": -503.1126403808594,
      "logps/rejected": -574.1822509765625,
      "loss": 0.4992,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.4480373859405518,
      "rewards/margins": 0.874676525592804,
      "rewards/rejected": -3.322713851928711,
      "step": 500
    },
    {
      "epoch": 0.7849293563579278,
      "eval_logits/chosen": -1.2588800191879272,
      "eval_logits/rejected": -1.1520029306411743,
      "eval_logps/chosen": -491.8629455566406,
      "eval_logps/rejected": -549.8136596679688,
      "eval_loss": 0.5018974542617798,
      "eval_rewards/accuracies": 0.7053571343421936,
      "eval_rewards/chosen": -2.332996129989624,
      "eval_rewards/margins": 0.7699158191680908,
      "eval_rewards/rejected": -3.102912187576294,
      "eval_runtime": 219.9359,
      "eval_samples_per_second": 9.094,
      "eval_steps_per_second": 0.382,
      "step": 500
    },
    {
      "epoch": 0.8006279434850864,
      "grad_norm": 4.936287879943848,
      "learning_rate": 5.819552438686238e-07,
      "logits/chosen": -1.2254358530044556,
      "logits/rejected": -1.1503589153289795,
      "logps/chosen": -488.61053466796875,
      "logps/rejected": -558.0599975585938,
      "loss": 0.4946,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.1799943447113037,
      "rewards/margins": 0.8831037282943726,
      "rewards/rejected": -3.063098192214966,
      "step": 510
    },
    {
      "epoch": 0.8163265306122449,
      "grad_norm": 4.393782615661621,
      "learning_rate": 4.969679867292276e-07,
      "logits/chosen": -1.4334200620651245,
      "logits/rejected": -1.2100530862808228,
      "logps/chosen": -525.8604736328125,
      "logps/rejected": -564.2894897460938,
      "loss": 0.5004,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.246795177459717,
      "rewards/margins": 0.9003856778144836,
      "rewards/rejected": -3.1471807956695557,
      "step": 520
    },
    {
      "epoch": 0.8320251177394035,
      "grad_norm": 4.474452495574951,
      "learning_rate": 4.1800035681877765e-07,
      "logits/chosen": -1.165138840675354,
      "logits/rejected": -1.085632085800171,
      "logps/chosen": -455.230712890625,
      "logps/rejected": -550.22412109375,
      "loss": 0.5097,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.193859338760376,
      "rewards/margins": 0.9264294505119324,
      "rewards/rejected": -3.120288372039795,
      "step": 530
    },
    {
      "epoch": 0.847723704866562,
      "grad_norm": 4.0765767097473145,
      "learning_rate": 3.4528967220911287e-07,
      "logits/chosen": -1.276033639907837,
      "logits/rejected": -1.2467362880706787,
      "logps/chosen": -488.040283203125,
      "logps/rejected": -554.4703979492188,
      "loss": 0.5123,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -2.10200834274292,
      "rewards/margins": 0.8350952863693237,
      "rewards/rejected": -2.937103748321533,
      "step": 540
    },
    {
      "epoch": 0.8634222919937206,
      "grad_norm": 4.756001949310303,
      "learning_rate": 2.7905444723949765e-07,
      "logits/chosen": -1.2069323062896729,
      "logits/rejected": -1.192297101020813,
      "logps/chosen": -484.1224060058594,
      "logps/rejected": -570.1021118164062,
      "loss": 0.5144,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.127053737640381,
      "rewards/margins": 0.9687838554382324,
      "rewards/rejected": -3.095837354660034,
      "step": 550
    },
    {
      "epoch": 0.8791208791208791,
      "grad_norm": 4.719667911529541,
      "learning_rate": 2.1949373582475065e-07,
      "logits/chosen": -1.2955992221832275,
      "logits/rejected": -1.2129268646240234,
      "logps/chosen": -453.64794921875,
      "logps/rejected": -541.0159912109375,
      "loss": 0.4979,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.9799339771270752,
      "rewards/margins": 1.0608704090118408,
      "rewards/rejected": -3.040804386138916,
      "step": 560
    },
    {
      "epoch": 0.8948194662480377,
      "grad_norm": 5.24905252456665,
      "learning_rate": 1.667865332469379e-07,
      "logits/chosen": -1.3544114828109741,
      "logits/rejected": -1.2046071290969849,
      "logps/chosen": -456.50482177734375,
      "logps/rejected": -518.5260620117188,
      "loss": 0.4768,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.8672730922698975,
      "rewards/margins": 1.0792367458343506,
      "rewards/rejected": -2.946509838104248,
      "step": 570
    },
    {
      "epoch": 0.9105180533751962,
      "grad_norm": 5.04511833190918,
      "learning_rate": 1.2109123822844653e-07,
      "logits/chosen": -1.314582109451294,
      "logits/rejected": -1.1942718029022217,
      "logps/chosen": -491.97967529296875,
      "logps/rejected": -532.6231689453125,
      "loss": 0.4948,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -2.1609413623809814,
      "rewards/margins": 0.8695161938667297,
      "rewards/rejected": -3.0304577350616455,
      "step": 580
    },
    {
      "epoch": 0.9262166405023547,
      "grad_norm": 4.632051467895508,
      "learning_rate": 8.254517690300946e-08,
      "logits/chosen": -1.3019325733184814,
      "logits/rejected": -1.1224420070648193,
      "logps/chosen": -463.02874755859375,
      "logps/rejected": -560.708251953125,
      "loss": 0.4826,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.165086507797241,
      "rewards/margins": 1.022458791732788,
      "rewards/rejected": -3.1875455379486084,
      "step": 590
    },
    {
      "epoch": 0.9419152276295133,
      "grad_norm": 5.119118690490723,
      "learning_rate": 5.126419011529993e-08,
      "logits/chosen": -1.251976490020752,
      "logits/rejected": -1.1646479368209839,
      "logps/chosen": -498.5111389160156,
      "logps/rejected": -600.1183471679688,
      "loss": 0.5001,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.133627414703369,
      "rewards/margins": 1.0616191625595093,
      "rewards/rejected": -3.195246696472168,
      "step": 600
    },
    {
      "epoch": 0.9419152276295133,
      "eval_logits/chosen": -1.2825318574905396,
      "eval_logits/rejected": -1.1767752170562744,
      "eval_logps/chosen": -475.6836853027344,
      "eval_logps/rejected": -535.6056518554688,
      "eval_loss": 0.5020818114280701,
      "eval_rewards/accuracies": 0.7083333134651184,
      "eval_rewards/chosen": -2.171203136444092,
      "eval_rewards/margins": 0.7896292805671692,
      "eval_rewards/rejected": -2.960832118988037,
      "eval_runtime": 219.3018,
      "eval_samples_per_second": 9.12,
      "eval_steps_per_second": 0.383,
      "step": 600
    },
    {
      "epoch": 0.957613814756672,
      "grad_norm": 5.81568717956543,
      "learning_rate": 2.734228528934679e-08,
      "logits/chosen": -1.3186975717544556,
      "logits/rejected": -1.1690412759780884,
      "logps/chosen": -448.78814697265625,
      "logps/rejected": -523.2693481445312,
      "loss": 0.5127,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.0488333702087402,
      "rewards/margins": 1.0742355585098267,
      "rewards/rejected": -3.1230690479278564,
      "step": 610
    },
    {
      "epoch": 0.9733124018838305,
      "grad_norm": 5.066900253295898,
      "learning_rate": 1.0851353912008644e-08,
      "logits/chosen": -1.1879260540008545,
      "logits/rejected": -1.057185411453247,
      "logps/chosen": -493.63323974609375,
      "logps/rejected": -551.7728271484375,
      "loss": 0.4958,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -2.1862809658050537,
      "rewards/margins": 0.9377900958061218,
      "rewards/rejected": -3.1240711212158203,
      "step": 620
    },
    {
      "epoch": 0.989010989010989,
      "grad_norm": 5.623207092285156,
      "learning_rate": 1.8409554805329243e-09,
      "logits/chosen": -1.2257732152938843,
      "logits/rejected": -1.1151111125946045,
      "logps/chosen": -482.53857421875,
      "logps/rejected": -528.3353271484375,
      "loss": 0.4897,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.206721544265747,
      "rewards/margins": 0.8017439842224121,
      "rewards/rejected": -3.008465528488159,
      "step": 630
    },
    {
      "epoch": 1.0,
      "step": 637,
      "total_flos": 0.0,
      "train_loss": 0.5397420258873861,
      "train_runtime": 14909.9591,
      "train_samples_per_second": 4.1,
      "train_steps_per_second": 0.043
    }
  ],
  "logging_steps": 10,
  "max_steps": 637,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}