| --model_name_or_path /data/public_models/Meta-Llama-3-8B --output_dir /data/yujia_wang/fedmoe_llama/moe/checkpoints_moe/Meta-Llama-3-8B/TIGER-Lab/MathInstruct/3e-6_noniid_layer_seq_2 --dataset_name TIGER-Lab/MathInstruct --dataset_config_name default --per_device_train_batch_size 1 --per_device_eval_batch_size 8 --num_train_epochs 1 --overwrite_output_dir --do_train True --do_eval --seed 42 --dataloader_num_workers 16 --disable_tqdm False --save_strategy no --evaluation_strategy epoch --load_best_model_at_end True --learning_rate 3e-6 --notes layer_seq_2 --split_strategy noniid --num_rounds 320 --num_clients 10 --sample_clients 1 --max_gate_samples 50 --max_train_samples 50000 --gradient_accumulation_steps 4 | |