| { | |
| "train_config": "config/is_selfssm.swe_2048.json", | |
| "model_name": "Selfssm", | |
| "devices": 8, | |
| "train_name": "is_selfssm.swe_2048", | |
| "name": "is_selfssm.swe_2048_Selfssm", | |
| "model_type": "selfssm", | |
| "nodes": 1, | |
| "block_size": 4096, | |
| "max_tokens": 20000000000, | |
| "global_batch_size": 512, | |
| "micro_batch_size": 2, | |
| "batch_size": 64, | |
| "gradient_accumulation_steps": 32, | |
| "learning_rate": 0.0004, | |
| "total_evals": 400, | |
| "warmup_tokens": 200000000, | |
| "eval_iters": 200, | |
| "resume_ckpt": "token-3775135744.iter-014400-ckpt.pth", | |
| "log_step_interval": 10, | |
| "save_step_interval": 300, | |
| "eval_step_interval": 300, | |
| "num_extrapol": 4, | |
| "weight_decay": 0.1, | |
| "beta1": 0.9, | |
| "beta2": 0.95, | |
| "grad_clip": 1.0, | |
| "decay_lr": true, | |
| "min_lr": 4e-05, | |
| "attn_type": "flash_attention_2", | |
| "attn_window_size": 2048, | |
| "output_attentions": false, | |
| "output_hidden_states": false, | |
| "load_linear_data": "next", | |
| "load_linear_overlap": 0, | |
| "load_linear_also_copy_rate": 0, | |
| "data_do_shuffle": false, | |
| "load_input_data": "", | |
| "log_iter_interval": 320, | |
| "tok": "llama2_tok", | |
| "transformer_hidden_size": 1024, | |
| "transformer_num_hidden_layers": 12, | |
| "transformer_intermediate_size": 4096, | |
| "linear_hidden_states": 2048, | |
| "linear_cal_loss": false, | |
| "linear_num_hidden_layers": 12, | |
| "linear_hidden_name": "output_embedding", | |
| "linear_multiply_self_attn_B": false, | |
| "linear_input_w_selfattn": false, | |
| "integrate_self_attn_weight": "", | |
| "integrate_self_attn_weight_rate": 1, | |
| "tie_word_embedding_layer": true, | |
| "linear_up_project": false, | |
| "linear_up_project_size": 0, | |
| "interact_mode": "cross_attn" | |
| } |