Outer Step 0. Inner Step 16. Batch Size 124

config.json

@@ -268,26 +268,24 @@
     "AutoModelForCausalLM": "distributed/optimized-gpt2-500m--modeling_gpt_optimized.GPTOptim"
   },
   "block_list": [
-    5129440,
-    5129485,
-    5129500,
-    5129512,
-    5129530,
-    5129537,
-    5129554,
-    5129565,
-    5129572,
-    5129585,
-    5129602,
-    5129615,
-    5129634
+    5129671,
+    5129676,
+    5129686,
+    5129693,
+    5129703,
+    5129711,
+    5129724,
+    5129736,
+    5129742,
+    5129751,
+    5129876
   ],
   "block_size": 1024,
   "bos_token_id": 50256,
   "embd_pdrop": 0.1,
   "eos_token_id": 50256,
   "initializer_range": 0.02,
-  "inner_step": 7,
+  "inner_step": 16,
   "inner_steps": 0,
   "last_allreduce_block": 5072405,
   "layer_norm_epsilon": 1e-05,

inner_optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9c45291ae2ec6ffdd05a013019049466c41c655d45c36d14069c4895958112ee
+oid sha256:5729c7008dd92d0b8057c52f1f153197a4f572604144aedaeb01cdea423b519b
 size 8081781770

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1cc6a61dde395205a774dbb97f4c7cbec743de65a9d6a85ec11057cf0f113d7b
+oid sha256:49ae8d63d75662b9179811d57f13887a67c3db2cf42433855770d2441370d3c2
 size 4040701744