{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.20181634712411706,
  "eval_steps": 50,
  "global_step": 250,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.008072653884964682,
      "grad_norm": 0.04926518723368645,
      "learning_rate": 4.999451708687114e-06,
      "logits/chosen": 14.542106628417969,
      "logits/rejected": 14.864250183105469,
      "logps/chosen": -0.2809702754020691,
      "logps/rejected": -0.3013763725757599,
      "loss": 0.9221,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.421455442905426,
      "rewards/margins": 0.03060910664498806,
      "rewards/rejected": -0.452064573764801,
      "step": 10
    },
    {
      "epoch": 0.016145307769929364,
      "grad_norm": 0.056249432265758514,
      "learning_rate": 4.997807075247147e-06,
      "logits/chosen": 14.614748001098633,
      "logits/rejected": 15.259109497070312,
      "logps/chosen": -0.2828002870082855,
      "logps/rejected": -0.3477819561958313,
      "loss": 0.916,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.4242004454135895,
      "rewards/margins": 0.09747247397899628,
      "rewards/rejected": -0.5216729044914246,
      "step": 20
    },
    {
      "epoch": 0.024217961654894045,
      "grad_norm": 0.0664869099855423,
      "learning_rate": 4.9950668210706795e-06,
      "logits/chosen": 14.391204833984375,
      "logits/rejected": 14.82734203338623,
      "logps/chosen": -0.287629634141922,
      "logps/rejected": -0.3329126834869385,
      "loss": 0.914,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.4314444661140442,
      "rewards/margins": 0.06792456656694412,
      "rewards/rejected": -0.4993689954280853,
      "step": 30
    },
    {
      "epoch": 0.03229061553985873,
      "grad_norm": 0.055584829300642014,
      "learning_rate": 4.9912321481237616e-06,
      "logits/chosen": 13.93278980255127,
      "logits/rejected": 14.886846542358398,
      "logps/chosen": -0.28155946731567383,
      "logps/rejected": -0.3678051829338074,
      "loss": 0.9273,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.42233920097351074,
      "rewards/margins": 0.12936851382255554,
      "rewards/rejected": -0.5517078042030334,
      "step": 40
    },
    {
      "epoch": 0.04036326942482341,
      "grad_norm": 0.07997103035449982,
      "learning_rate": 4.986304738420684e-06,
      "logits/chosen": 14.172693252563477,
      "logits/rejected": 14.742494583129883,
      "logps/chosen": -0.28225988149642944,
      "logps/rejected": -0.34329456090927124,
      "loss": 0.9081,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.42338982224464417,
      "rewards/margins": 0.0915520042181015,
      "rewards/rejected": -0.5149418115615845,
      "step": 50
    },
    {
      "epoch": 0.04036326942482341,
      "eval_logits/chosen": 14.200166702270508,
      "eval_logits/rejected": 14.817726135253906,
      "eval_logps/chosen": -0.2625390887260437,
      "eval_logps/rejected": -0.3458769917488098,
      "eval_loss": 0.9080610275268555,
      "eval_rewards/accuracies": 0.5544554591178894,
      "eval_rewards/chosen": -0.39380866289138794,
      "eval_rewards/margins": 0.1250067949295044,
      "eval_rewards/rejected": -0.5188154578208923,
      "eval_runtime": 29.8098,
      "eval_samples_per_second": 26.87,
      "eval_steps_per_second": 3.388,
      "step": 50
    },
    {
      "epoch": 0.04843592330978809,
      "grad_norm": 0.06322095543146133,
      "learning_rate": 4.980286753286196e-06,
      "logits/chosen": 14.008010864257812,
      "logits/rejected": 14.975939750671387,
      "logps/chosen": -0.2697351574897766,
      "logps/rejected": -0.3445274233818054,
      "loss": 0.9195,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.40460270643234253,
      "rewards/margins": 0.1121884360909462,
      "rewards/rejected": -0.5167912244796753,
      "step": 60
    },
    {
      "epoch": 0.056508577194752774,
      "grad_norm": 0.12975476682186127,
      "learning_rate": 4.973180832407471e-06,
      "logits/chosen": 14.213134765625,
      "logits/rejected": 14.889978408813477,
      "logps/chosen": -0.2839818596839905,
      "logps/rejected": -0.3382417559623718,
      "loss": 0.9241,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.4259727895259857,
      "rewards/margins": 0.08138985931873322,
      "rewards/rejected": -0.5073626637458801,
      "step": 70
    },
    {
      "epoch": 0.06458123107971746,
      "grad_norm": 0.08352109789848328,
      "learning_rate": 4.964990092676263e-06,
      "logits/chosen": 14.395146369934082,
      "logits/rejected": 14.900177001953125,
      "logps/chosen": -0.27067264914512634,
      "logps/rejected": -0.3516673743724823,
      "loss": 0.9251,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.4060089588165283,
      "rewards/margins": 0.12149210274219513,
      "rewards/rejected": -0.5275009870529175,
      "step": 80
    },
    {
      "epoch": 0.07265388496468214,
      "grad_norm": 0.0793827548623085,
      "learning_rate": 4.9557181268217225e-06,
      "logits/chosen": 13.905145645141602,
      "logits/rejected": 14.897878646850586,
      "logps/chosen": -0.26814645528793335,
      "logps/rejected": -0.38061466813087463,
      "loss": 0.9055,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4022197127342224,
      "rewards/margins": 0.16870227456092834,
      "rewards/rejected": -0.5709219574928284,
      "step": 90
    },
    {
      "epoch": 0.08072653884964683,
      "grad_norm": 0.09212008118629456,
      "learning_rate": 4.9453690018345144e-06,
      "logits/chosen": 14.11627197265625,
      "logits/rejected": 14.571977615356445,
      "logps/chosen": -0.3130624294281006,
      "logps/rejected": -0.35073375701904297,
      "loss": 0.911,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.46959367394447327,
      "rewards/margins": 0.05650699883699417,
      "rewards/rejected": -0.5261006951332092,
      "step": 100
    },
    {
      "epoch": 0.08072653884964683,
      "eval_logits/chosen": 13.845876693725586,
      "eval_logits/rejected": 14.490789413452148,
      "eval_logps/chosen": -0.2531408667564392,
      "eval_logps/rejected": -0.3464036285877228,
      "eval_loss": 0.8986235857009888,
      "eval_rewards/accuracies": 0.5544554591178894,
      "eval_rewards/chosen": -0.3797112703323364,
      "eval_rewards/margins": 0.13989417254924774,
      "eval_rewards/rejected": -0.5196054577827454,
      "eval_runtime": 29.0914,
      "eval_samples_per_second": 27.534,
      "eval_steps_per_second": 3.472,
      "step": 100
    },
    {
      "epoch": 0.08879919273461151,
      "grad_norm": 0.08554862439632416,
      "learning_rate": 4.933947257182901e-06,
      "logits/chosen": 13.416229248046875,
      "logits/rejected": 14.582674026489258,
      "logps/chosen": -0.2523443102836609,
      "logps/rejected": -0.38751405477523804,
      "loss": 0.8993,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3785164952278137,
      "rewards/margins": 0.20275457203388214,
      "rewards/rejected": -0.5812710523605347,
      "step": 110
    },
    {
      "epoch": 0.09687184661957618,
      "grad_norm": 0.29209578037261963,
      "learning_rate": 4.921457902821578e-06,
      "logits/chosen": 13.874654769897461,
      "logits/rejected": 14.423624992370605,
      "logps/chosen": -0.27300480008125305,
      "logps/rejected": -0.3526575267314911,
      "loss": 0.8942,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.40950721502304077,
      "rewards/margins": 0.11947910487651825,
      "rewards/rejected": -0.5289863348007202,
      "step": 120
    },
    {
      "epoch": 0.10494450050454086,
      "grad_norm": 0.1028478816151619,
      "learning_rate": 4.907906416994146e-06,
      "logits/chosen": 14.030723571777344,
      "logits/rejected": 14.711235046386719,
      "logps/chosen": -0.27410784363746643,
      "logps/rejected": -0.3665519058704376,
      "loss": 0.8922,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.41116175055503845,
      "rewards/margins": 0.1386660784482956,
      "rewards/rejected": -0.5498278737068176,
      "step": 130
    },
    {
      "epoch": 0.11301715438950555,
      "grad_norm": 0.08459590375423431,
      "learning_rate": 4.893298743830168e-06,
      "logits/chosen": 13.477182388305664,
      "logits/rejected": 13.919464111328125,
      "logps/chosen": -0.2495063841342926,
      "logps/rejected": -0.3594816029071808,
      "loss": 0.8957,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.3742595613002777,
      "rewards/margins": 0.16496284306049347,
      "rewards/rejected": -0.5392224192619324,
      "step": 140
    },
    {
      "epoch": 0.12108980827447023,
      "grad_norm": 0.12467797845602036,
      "learning_rate": 4.8776412907378845e-06,
      "logits/chosen": 12.801656723022461,
      "logits/rejected": 13.564155578613281,
      "logps/chosen": -0.2779986262321472,
      "logps/rejected": -0.33566445112228394,
      "loss": 0.892,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4169979691505432,
      "rewards/margins": 0.08649872243404388,
      "rewards/rejected": -0.5034967064857483,
      "step": 150
    },
    {
      "epoch": 0.12108980827447023,
      "eval_logits/chosen": 12.169118881225586,
      "eval_logits/rejected": 13.019683837890625,
      "eval_logps/chosen": -0.25547870993614197,
      "eval_logps/rejected": -0.37277930974960327,
      "eval_loss": 0.8789658546447754,
      "eval_rewards/accuracies": 0.5742574334144592,
      "eval_rewards/chosen": -0.38321802020072937,
      "eval_rewards/margins": 0.17595094442367554,
      "eval_rewards/rejected": -0.5591689944267273,
      "eval_runtime": 29.0961,
      "eval_samples_per_second": 27.529,
      "eval_steps_per_second": 3.471,
      "step": 150
    },
    {
      "epoch": 0.12916246215943492,
      "grad_norm": 0.1712370663881302,
      "learning_rate": 4.860940925593703e-06,
      "logits/chosen": 11.972528457641602,
      "logits/rejected": 12.684088706970215,
      "logps/chosen": -0.28340521454811096,
      "logps/rejected": -0.3864063024520874,
      "loss": 0.8789,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.42510780692100525,
      "rewards/margins": 0.15450166165828705,
      "rewards/rejected": -0.5796095132827759,
      "step": 160
    },
    {
      "epoch": 0.13723511604439959,
      "grad_norm": 0.21750673651695251,
      "learning_rate": 4.84320497372973e-06,
      "logits/chosen": 11.93881607055664,
      "logits/rejected": 12.458230972290039,
      "logps/chosen": -0.274599552154541,
      "logps/rejected": -0.3882916271686554,
      "loss": 0.8857,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4118993282318115,
      "rewards/margins": 0.1705380380153656,
      "rewards/rejected": -0.5824374556541443,
      "step": 170
    },
    {
      "epoch": 0.14530776992936428,
      "grad_norm": 0.16000741720199585,
      "learning_rate": 4.824441214720629e-06,
      "logits/chosen": 9.631464958190918,
      "logits/rejected": 10.772969245910645,
      "logps/chosen": -0.287865549325943,
      "logps/rejected": -0.4817379415035248,
      "loss": 0.8699,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.4317983090877533,
      "rewards/margins": 0.29080861806869507,
      "rewards/rejected": -0.722606897354126,
      "step": 180
    },
    {
      "epoch": 0.15338042381432895,
      "grad_norm": 0.46293890476226807,
      "learning_rate": 4.804657878971252e-06,
      "logits/chosen": 9.081937789916992,
      "logits/rejected": 10.024572372436523,
      "logps/chosen": -0.2941994071006775,
      "logps/rejected": -0.4772109389305115,
      "loss": 0.8565,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.44129911065101624,
      "rewards/margins": 0.2745172679424286,
      "rewards/rejected": -0.7158163785934448,
      "step": 190
    },
    {
      "epoch": 0.16145307769929365,
      "grad_norm": 0.24658434092998505,
      "learning_rate": 4.783863644106502e-06,
      "logits/chosen": 7.767125606536865,
      "logits/rejected": 8.009145736694336,
      "logps/chosen": -0.32877305150032043,
      "logps/rejected": -0.47733697295188904,
      "loss": 0.8482,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.49315959215164185,
      "rewards/margins": 0.2228458821773529,
      "rewards/rejected": -0.7160054445266724,
      "step": 200
    },
    {
      "epoch": 0.16145307769929365,
      "eval_logits/chosen": 6.716187953948975,
      "eval_logits/rejected": 7.248146057128906,
      "eval_logps/chosen": -0.30652713775634766,
      "eval_logps/rejected": -0.5056277513504028,
      "eval_loss": 0.8243693113327026,
      "eval_rewards/accuracies": 0.603960394859314,
      "eval_rewards/chosen": -0.4597907066345215,
      "eval_rewards/margins": 0.2986510097980499,
      "eval_rewards/rejected": -0.7584417462348938,
      "eval_runtime": 29.0989,
      "eval_samples_per_second": 27.527,
      "eval_steps_per_second": 3.471,
      "step": 200
    },
    {
      "epoch": 0.16952573158425832,
      "grad_norm": 0.40796443819999695,
      "learning_rate": 4.762067631165049e-06,
      "logits/chosen": 5.932644844055176,
      "logits/rejected": 6.521953582763672,
      "logps/chosen": -0.32976508140563965,
      "logps/rejected": -0.5628186464309692,
      "loss": 0.7988,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.4946475923061371,
      "rewards/margins": 0.3495803475379944,
      "rewards/rejected": -0.8442279696464539,
      "step": 210
    },
    {
      "epoch": 0.17759838546922302,
      "grad_norm": 0.42468318343162537,
      "learning_rate": 4.7392794005985324e-06,
      "logits/chosen": 6.069305896759033,
      "logits/rejected": 5.8950395584106445,
      "logps/chosen": -0.37205421924591064,
      "logps/rejected": -0.6190425753593445,
      "loss": 0.796,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5580812692642212,
      "rewards/margins": 0.3704826235771179,
      "rewards/rejected": -0.9285639524459839,
      "step": 220
    },
    {
      "epoch": 0.1856710393541877,
      "grad_norm": 0.4138280153274536,
      "learning_rate": 4.715508948078037e-06,
      "logits/chosen": 3.5271706581115723,
      "logits/rejected": 3.363534927368164,
      "logps/chosen": -0.41677650809288025,
      "logps/rejected": -0.7121980786323547,
      "loss": 0.7457,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.6251648664474487,
      "rewards/margins": 0.4431324005126953,
      "rewards/rejected": -1.0682971477508545,
      "step": 230
    },
    {
      "epoch": 0.19374369323915236,
      "grad_norm": 1.4327284097671509,
      "learning_rate": 4.690766700109659e-06,
      "logits/chosen": 2.296924114227295,
      "logits/rejected": 1.6135867834091187,
      "logps/chosen": -0.506227970123291,
      "logps/rejected": -0.8080593943595886,
      "loss": 0.7453,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.7593418955802917,
      "rewards/margins": 0.4527471661567688,
      "rewards/rejected": -1.21208918094635,
      "step": 240
    },
    {
      "epoch": 0.20181634712411706,
      "grad_norm": 0.4413074553012848,
      "learning_rate": 4.665063509461098e-06,
      "logits/chosen": 1.4450469017028809,
      "logits/rejected": 0.40727120637893677,
      "logps/chosen": -0.4749869406223297,
      "logps/rejected": -0.9967275857925415,
      "loss": 0.704,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.712480366230011,
      "rewards/margins": 0.7826108932495117,
      "rewards/rejected": -1.4950913190841675,
      "step": 250
    },
    {
      "epoch": 0.20181634712411706,
      "eval_logits/chosen": 1.4614256620407104,
      "eval_logits/rejected": 0.6895493865013123,
      "eval_logps/chosen": -0.5038881301879883,
      "eval_logps/rejected": -0.973581850528717,
      "eval_loss": 0.6982013583183289,
      "eval_rewards/accuracies": 0.6138613820075989,
      "eval_rewards/chosen": -0.7558321952819824,
      "eval_rewards/margins": 0.7045406699180603,
      "eval_rewards/rejected": -1.460372805595398,
      "eval_runtime": 29.0993,
      "eval_samples_per_second": 27.526,
      "eval_steps_per_second": 3.471,
      "step": 250
    }
  ],
  "logging_steps": 10,
  "max_steps": 1500,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 5.813983197213491e+17,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}