Outer Step 4. Inner Step 177. Batch Size 360

config.json

@@ -268,23 +268,23 @@
     "AutoModelForCausalLM": "distributed/optimized-gpt2-500m--modeling_gpt_optimized.GPTOptim"
   },
   "block_list": [
-    5150155,
-    5150159,
-    5150163,
-    5150167,
-    5150171,
-    5150175,
-    5150179,
-    5150183,
-    5150187,
-    5150191
+    5150220,
+    5150224,
+    5150228,
+    5150232,
+    5150236,
+    5150240,
+    5150244,
+    5150248,
+    5150252,
+    5150256
   ],
   "block_size": 1024,
   "bos_token_id": 50256,
   "embd_pdrop": 0.1,
   "eos_token_id": 50256,
   "initializer_range": 0.02,
-  "inner_step": 168,
+  "inner_step": 177,
   "inner_steps": 0,
   "last_allreduce_block": 5148895,
   "layer_norm_epsilon": 1e-05,

inner_optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:74b26d60b235da5b1d7f7b3ab71d6ecf3097907bc3fedbd7226610a8b6bf1439
+oid sha256:9154a9e0411f199fb2d971a74e2c1c1d37819b63998ef97f839f93414b71bc73
 size 8081781770

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2aee6de99d39dcd1452919e363f845b3b3939204e1e24fb7bf32cdeab964aca3
+oid sha256:253939b9adf061e93ef2df934ea815d3853804f85d8d29710eecebaf303a5a53
 size 4040701744