Add TB sbatch

sbatch_4b284b84b10c4pyseed1.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b10c4pyseed1
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b10c4py.txt
+# "train: 0.1 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.9 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 1 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b10c4pyseed2.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b10c4pyseed2
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b10c4py.txt
+# "train: 0.1 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.9 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 2 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b10c4pyseed3.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b10c4pyseed3
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b10c4py.txt
+# "train: 0.1 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.9 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 3 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b10c4pyseed4.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b10c4pyseed4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b10c4py.txt
+# "train: 0.1 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.9 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 4 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b20c4pyseed1.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b20c4pyseed1
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b20c4py.txt
+# "train: 0.2 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.8 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 1 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b20c4pyseed2.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b20c4pyseed2
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b20c4py.txt
+# "train: 0.2 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.8 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 2 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b20c4pyseed3.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b20c4pyseed3
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b20c4py.txt
+# "train: 0.2 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.8 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 3 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b20c4pyseed4.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b20c4pyseed4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b20c4py.txt
+# "train: 0.2 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.8 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 4 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b30c4pyseed1.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b30c4pyseed1
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b30c4py.txt
+# "train: 0.3 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.7 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 1 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b30c4pyseed2.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b30c4pyseed2
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b30c4py.txt
+# "train: 0.3 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.7 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 2 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b30c4pyseed3.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b30c4pyseed3
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b30c4py.txt
+# "train: 0.3 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.7 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 3 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b30c4pyseed4.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b30c4pyseed4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b30c4py.txt
+# "train: 0.3 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.7 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 4 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b40c4pyseed1.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b40c4pyseed1
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b40c4py.txt
+# "train: 0.4 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.6 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 1 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b40c4pyseed2.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b40c4pyseed2
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b40c4py.txt
+# "train: 0.4 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.6 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 2 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b40c4pyseed3.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b40c4pyseed3
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b40c4py.txt
+# "train: 0.4 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.6 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 3 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b40c4pyseed4.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b40c4pyseed4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b40c4py.txt
+# "train: 0.4 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.6 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 4 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b50c4pyseed1.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b50c4pyseed1
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b50c4py.txt
+# "train: 0.5 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.5 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 1 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b50c4pyseed2.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b50c4pyseed2
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b50c4py.txt
+# "train: 0.5 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.5 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 2 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b50c4pyseed3.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b50c4pyseed3
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b50c4py.txt
+# "train: 0.5 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.5 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 3 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b50c4pyseed4.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b50c4pyseed4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b50c4py.txt
+# "train: 0.5 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.5 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 4 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b60c4pyseed1.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b60c4pyseed1
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b60c4py.txt
+# "train: 0.6 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.4 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 1 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b60c4pyseed2.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b60c4pyseed2
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b60c4py.txt
+# "train: 0.6 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.4 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 2 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b60c4pyseed3.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b60c4pyseed3
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b60c4py.txt
+# "train: 0.6 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.4 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 3 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b60c4pyseed4.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b60c4pyseed4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b60c4py.txt
+# "train: 0.6 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.4 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 4 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b70c4pyseed1.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b70c4pyseed1
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b70c4py.txt
+# "train: 0.7 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.3 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 1 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b70c4pyseed2.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b70c4pyseed2
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b70c4py.txt
+# "train: 0.7 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.3 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 2 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b70c4pyseed3.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b70c4pyseed3
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b70c4py.txt
+# "train: 0.7 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.3 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 3 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b70c4pyseed4.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b70c4pyseed4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b70c4py.txt
+# "train: 0.7 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.3 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 4 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b80c4pyseed1.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b80c4pyseed1
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b80c4py.txt
+# "train: 0.8 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.2 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 1 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b80c4pyseed2.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b80c4pyseed2
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b80c4py.txt
+# "train: 0.8 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.2 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 2 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b80c4pyseed3.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b80c4pyseed3
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b80c4py.txt
+# "train: 0.8 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.2 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 3 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b80c4pyseed4.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b80c4pyseed4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b80c4py.txt
+# "train: 0.8 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.2 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 4 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b90c4pyseed1.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b90c4pyseed1
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b90c4py.txt
+# "train: 0.9 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.1 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 1 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b90c4pyseed2.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b90c4pyseed2
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b90c4py.txt
+# "train: 0.9 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.1 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 2 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b90c4pyseed3.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b90c4pyseed3
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b90c4py.txt
+# "train: 0.9 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.1 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 3 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

sbatch_4b284b84b90c4pyseed4.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b84b90c4pyseed4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+
+TRAIN_DATA_PATH=train84b90c4py.txt
+# "train: 0.9 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document, 0.1 0:1 /scratch/project_462000119/data/python/gpt2tok_python_84B_content_document"
+VALID_DATA_PATH=valc4py.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_python: 1.0 0.95:1  /scratch/project_462000119/data/python/gpt2tok_python_content_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=10000
+
+# Tokens: 84_000_000_000
+# -> Samples: 41_015_625.0
+TRAIN_SAMPLES=41_015_625
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.95 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 410_156 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 5000 \
+    --eval-iters 10 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    --seed 4 \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

tensorboard/tensorboard_4b284b84b10c4pyseed1/events.out.tfevents.1683756022.nid007048.61333.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d67bc92ca2ba02857de441b9e358b7c0c23a3e7e4d20c822a50f4bb41c035023
+size 19996

tensorboard/tensorboard_4b284b84b10c4pyseed1/events.out.tfevents.1683756633.nid006995.89119.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0333176e62af9844e2b9390582b8da6291daa668b619e3105c0df66e4961820
+size 113224610

tensorboard/tensorboard_4b284b84b10c4pyseed1/events.out.tfevents.1683928600.nid007131.83743.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c5e6659617a6329ad7bdfe6575be5e875bdff89b84f293b3299f6c735079f4d
+size 30411691

tensorboard/tensorboard_4b284b84b10c4pyseed1/events.out.tfevents.1683974921.nid006671.116485.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c44a801835b83dc6284b98fee71d9599e2b78bddd36d063132446d76881a74f0
+size 40

tensorboard/tensorboard_4b284b84b10c4pyseed1/events.out.tfevents.1683978316.nid006500.80744.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5aad21bc9610e6b6c11f76b3b02aa534d7d3a6b8787b9f3ba60a32cb5fd98db
+size 216295

tensorboard/tensorboard_4b284b84b10c4pyseed2/events.out.tfevents.1683756022.nid005878.51908.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a62915fb10088dff87555a23741c81f59215fb2f8c6e29c529d3858a7f393696
+size 113531666

tensorboard/tensorboard_4b284b84b10c4pyseed2/events.out.tfevents.1683928065.nid006518.16895.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0736d9395b5347605e7b441f2eae463bc34465241b388aa5fe225eb35ed502f1
+size 30104633

tensorboard/tensorboard_4b284b84b10c4pyseed2/events.out.tfevents.1683973531.nid007019.108443.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f873da0be24a74805be74b1468d4068c6196cedcf95532dd401243b8e463e4c
+size 22933

tensorboard/tensorboard_4b284b84b10c4pyseed2/events.out.tfevents.1683978316.nid006848.10586.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39618b3ba29c6ecf0c6a27d79d2a9a07f1afbc2010f478accc507e7e4bdad8f4
+size 216295

tensorboard/tensorboard_4b284b84b10c4pyseed3/events.out.tfevents.1683756022.nid006518.29112.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f479f2724d0af83fbe9116768d933689c9102159cf721f25daed543a3a8c339e
+size 113815604

tensorboard/tensorboard_4b284b84b10c4pyseed3/events.out.tfevents.1683928065.nid005878.6854.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf920e80db0cc8fbdd8691551a242c53a2a4e9aaddd8a7cc5dea7e663d7af02d
+size 29820697

tensorboard/tensorboard_4b284b84b10c4pyseed3/events.out.tfevents.1683974293.nid006598.20858.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75e21748690b8a869904b54a47db62873fe4b586067ed450219a6b926f5b511d
+size 22933

tensorboard/tensorboard_4b284b84b10c4pyseed3/events.out.tfevents.1683978316.nid007019.25418.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e0b8256e9a31b5290f0eb83c30f3aa5fb604a96a61631d1b13667d5d2edbb1e
+size 216295

tensorboard/tensorboard_4b284b84b10c4pyseed4/events.out.tfevents.1683756022.nid006586.32254.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a32662873dc5f8d16b35c82bd2970c50aef0dc39a6a1b110a0b03c3dda84dca
+size 113637713