Saving weights and logs of step 20000

events.out.tfevents.1629902662.t1v-n-1ae8dadb-w-0.181842.0.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:830bfc14ddfb9e53f03a7c79030b4fc5ddee472ee809578fca2e3d227a8cc3c7
-size 1470757
+oid sha256:879dce3ba5ae79c251e40ed3545f5b62ab82171da42a73cfdb24025f4f0aead5
+size 2948827

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9c8274c4dd16c72c5f36f1e4441851a0dd44dd2f1fe464977bbf43174e6b6a12
+oid sha256:735818e94e7088035a440913311cb15c642e4898b07e3c2a408c461cc77a76a8
 size 711588089

start_train.sh

@@ -12,8 +12,11 @@ python3 run_mlm_flax.py \
     --preprocessing_num_workers="96" \
     --per_device_train_batch_size="64" \
     --per_device_eval_batch_size="64" \
-    --learning_rate="3e-4" \
-    --warmup_steps="1000" \
+    --adam_beta1="0.9" \
+    --adam_beta2="0.98" \
+    --adam_epsilon="1e-6" \
+    --learning_rate="4e-4" \
+    --warmup_steps="20000" \
     --overwrite_output_dir \
     --num_train_epochs="3" \
     --save_strategy="steps" \