Hugging Face's logo

datablations
/
lm1-2b8-55b-oscarseeds

Model card Files
xet
Community
Muennighoff commited on
Commit
e023504
·
1 Parent(s): 06f95a6

Add

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. 2b855b55boscarseed1/evaluation/2b855b55boscarseed1_0_babi.json +22 -0
  2. 2b855b55boscarseed1/evaluation/2b855b55boscarseed1_1_babi.json +22 -0
  3. 2b855b55boscarseed1/evaluation/2b855b55boscarseed1_2_babi.json +22 -0
  4. 2b855b55boscarseed1/evaluation/2b855b55boscarseed1_3_babi.json +22 -0
  5. 2b855b55boscarseed1/evaluation/2b855b55boscarseed1_4_babi.json +22 -0
  6. 2b855b55boscarseed1/evaluation/2b855b55boscarseed1_5_babi.json +22 -0
  7. 2b855b55boscarseed1/evaluation/2b855b55boscarseed2_0_babi.json +22 -0
  8. 2b855b55boscarseed1/evaluation/2b855b55boscarseed2_1_babi.json +22 -0
  9. 2b855b55boscarseed1/evaluation/2b855b55boscarseed2_2_babi.json +22 -0
  10. 2b855b55boscarseed1/evaluation/2b855b55boscarseed2_3_babi.json +22 -0
  11. 2b855b55boscarseed1/evaluation/2b855b55boscarseed2_4_babi.json +22 -0
  12. 2b855b55boscarseed1/evaluation/2b855b55boscarseed2_5_babi.json +22 -0
  13. 2b855b55boscarseed1/evaluation/2b855b55boscarseed3_0_babi.json +22 -0
  14. 2b855b55boscarseed1/evaluation/2b855b55boscarseed3_1_babi.json +22 -0
  15. 2b855b55boscarseed1/evaluation/2b855b55boscarseed3_2_babi.json +22 -0
  16. 2b855b55boscarseed1/evaluation/2b855b55boscarseed3_3_babi.json +22 -0
  17. 2b855b55boscarseed1/evaluation/2b855b55boscarseed3_4_babi.json +22 -0
  18. 2b855b55boscarseed1/evaluation/2b855b55boscarseed3_5_babi.json +22 -0
  19. 2b855b55boscarseed1/evaluation/2b855b55boscarseed4_0_babi.json +22 -0
  20. 2b855b55boscarseed1/evaluation/2b855b55boscarseed4_1_babi.json +22 -0
  21. 2b855b55boscarseed1/evaluation/2b855b55boscarseed4_2_babi.json +22 -0
  22. 2b855b55boscarseed1/evaluation/2b855b55boscarseed4_3_babi.json +22 -0
  23. 2b855b55boscarseed1/evaluation/2b855b55boscarseed4_4_babi.json +22 -0
  24. 2b855b55boscarseed1/evaluation/2b855b55boscarseed4_5_babi.json +22 -0
  25. sbatch_2b855b11boscarseed1.sh +166 -0
  26. sbatch_2b855b11boscarseed2.sh +166 -0
  27. sbatch_2b855b11boscarseed3.sh +166 -0
  28. sbatch_2b855b11boscarseed4.sh +166 -0
  29. sbatch_2b855b14boscarseed1.sh +166 -0
  30. sbatch_2b855b14boscarseed2.sh +166 -0
  31. sbatch_2b855b14boscarseed3.sh +166 -0
  32. sbatch_2b855b14boscarseed4.sh +166 -0
  33. sbatch_2b855b18boscarseed1.sh +166 -0
  34. sbatch_2b855b18boscarseed2.sh +166 -0
  35. sbatch_2b855b18boscarseed3.sh +166 -0
  36. sbatch_2b855b18boscarseed4.sh +166 -0
  37. sbatch_2b855b1b25oscarseed1.sh +166 -0
  38. sbatch_2b855b1b25oscarseed2.sh +166 -0
  39. sbatch_2b855b1b25oscarseed3.sh +166 -0
  40. sbatch_2b855b1b25oscarseed4.sh +166 -0
  41. sbatch_2b855b28boscarseed1.sh +166 -0
  42. sbatch_2b855b28boscarseed2.sh +166 -0
  43. sbatch_2b855b28boscarseed3.sh +166 -0
  44. sbatch_2b855b28boscarseed4.sh +166 -0
  45. sbatch_2b855b4boscarseed1.sh +166 -0
  46. sbatch_2b855b4boscarseed2.sh +166 -0
  47. sbatch_2b855b4boscarseed3.sh +166 -0
  48. sbatch_2b855b4boscarseed4.sh +166 -0
  49. sbatch_2b855b55boscarseed1.sh +166 -0
  50. sbatch_2b855b55boscarseed2.sh +166 -0
2b855b55boscarseed1/evaluation/2b855b55boscarseed1_0_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.0,
5
+ "em_stderr": 0.0
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed1/transformers",
14
+ "num_fewshot": 0,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed1_1_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.08,
5
+ "em_stderr": 0.004953939224288839
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed1/transformers",
14
+ "num_fewshot": 1,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed1_2_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.159,
5
+ "em_stderr": 0.006677414366420249
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed1/transformers",
14
+ "num_fewshot": 2,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed1_3_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.21933333333333332,
5
+ "em_stderr": 0.007556086214902187
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed1/transformers",
14
+ "num_fewshot": 3,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed1_4_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.25233333333333335,
5
+ "em_stderr": 0.007931459097913785
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed1/transformers",
14
+ "num_fewshot": 4,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed1_5_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.259,
5
+ "em_stderr": 0.007999645707393086
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed1/transformers",
14
+ "num_fewshot": 5,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed2_0_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.0,
5
+ "em_stderr": 0.0
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed2/transformers",
14
+ "num_fewshot": 0,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed2_1_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.072,
5
+ "em_stderr": 0.004720108735558263
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed2/transformers",
14
+ "num_fewshot": 1,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed2_2_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.14833333333333334,
5
+ "em_stderr": 0.006490317911631995
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed2/transformers",
14
+ "num_fewshot": 2,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed2_3_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.20233333333333334,
5
+ "em_stderr": 0.007335947375298662
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed2/transformers",
14
+ "num_fewshot": 3,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed2_4_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.22266666666666668,
5
+ "em_stderr": 0.007597015573408561
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed2/transformers",
14
+ "num_fewshot": 4,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed2_5_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.24833333333333332,
5
+ "em_stderr": 0.007889362689354152
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed2/transformers",
14
+ "num_fewshot": 5,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed3_0_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.0,
5
+ "em_stderr": 0.0
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed3/transformers",
14
+ "num_fewshot": 0,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed3_1_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.08,
5
+ "em_stderr": 0.004953939224288839
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed3/transformers",
14
+ "num_fewshot": 1,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed3_2_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.159,
5
+ "em_stderr": 0.006677414366420249
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed3/transformers",
14
+ "num_fewshot": 2,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed3_3_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.21933333333333332,
5
+ "em_stderr": 0.007556086214902187
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed3/transformers",
14
+ "num_fewshot": 3,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed3_4_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.25233333333333335,
5
+ "em_stderr": 0.007931459097913785
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed3/transformers",
14
+ "num_fewshot": 4,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed3_5_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.259,
5
+ "em_stderr": 0.007999645707393086
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed3/transformers",
14
+ "num_fewshot": 5,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed4_0_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.0,
5
+ "em_stderr": 0.0
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed4/transformers",
14
+ "num_fewshot": 0,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed4_1_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.068,
5
+ "em_stderr": 0.0045969965694086165
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed4/transformers",
14
+ "num_fewshot": 1,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed4_2_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.12166666666666667,
5
+ "em_stderr": 0.005969352898009278
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed4/transformers",
14
+ "num_fewshot": 2,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed4_3_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.17166666666666666,
5
+ "em_stderr": 0.006885846283046821
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed4/transformers",
14
+ "num_fewshot": 3,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed4_4_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.22,
5
+ "em_stderr": 0.007564328987051203
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed4/transformers",
14
+ "num_fewshot": 4,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
2b855b55boscarseed1/evaluation/2b855b55boscarseed4_5_babi.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "babi": {
4
+ "em": 0.23633333333333334,
5
+ "em_stderr": 0.007757578421815523
6
+ }
7
+ },
8
+ "versions": {
9
+ "babi": 0
10
+ },
11
+ "config": {
12
+ "model": "gpt2",
13
+ "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscarseeds/2b855b55boscarseed4/transformers",
14
+ "num_fewshot": 5,
15
+ "batch_size": null,
16
+ "device": null,
17
+ "no_cache": true,
18
+ "limit": 3000,
19
+ "bootstrap_iters": 100000,
20
+ "description_dict": {}
21
+ }
22
+ }
sbatch_2b855b11boscarseed1.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b11boscarseed1
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train11boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_11B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 1 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b11boscarseed2.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b11boscarseed2
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train11boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_11B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 2 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b11boscarseed3.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b11boscarseed3
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train11boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_11B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 3 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b11boscarseed4.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b11boscarseed4
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train11boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_11B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 4 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b14boscarseed1.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b14boscarseed1
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train14boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_14B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 1 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b14boscarseed2.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b14boscarseed2
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train14boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_14B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 2 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b14boscarseed3.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b14boscarseed3
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train14boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_14B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 3 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b14boscarseed4.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b14boscarseed4
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train14boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_14B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 4 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b18boscarseed1.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b18boscarseed1
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train18boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_18B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 1 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b18boscarseed2.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b18boscarseed2
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train18boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_18B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 2 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b18boscarseed3.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b18boscarseed3
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train18boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_18B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 3 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b18boscarseed4.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b18boscarseed4
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train18boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_18B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 4 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b1b25oscarseed1.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b1b25oscarseed1
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train1b25oscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_1B25_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 1 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b1b25oscarseed2.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b1b25oscarseed2
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train1b25oscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_1B25_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 2 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b1b25oscarseed3.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b1b25oscarseed3
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train1b25oscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_1B25_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 3 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b1b25oscarseed4.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b1b25oscarseed4
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train1b25oscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_1B25_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 4 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b28boscarseed1.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b28boscarseed1
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train28boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_28B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 1 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b28boscarseed2.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b28boscarseed2
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train28boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_28B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 2 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b28boscarseed3.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b28boscarseed3
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train28boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_28B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 3 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b28boscarseed4.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b28boscarseed4
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train28boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_28B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 4 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b4boscarseed1.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b4boscarseed1
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train4boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_4B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 1 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b4boscarseed2.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b4boscarseed2
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train4boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_4B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 2 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b4boscarseed3.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b4boscarseed3
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train4boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_4B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 3 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b4boscarseed4.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b4boscarseed4
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train4boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_4B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 4 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b55boscarseed1.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b55boscarseed1
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train55boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_55B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 1 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"
sbatch_2b855b55boscarseed2.sh ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=2b855b55boscarseed2
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+ #TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
39
+ #VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
40
+
41
+ TRAIN_DATA_PATH=train55boscar.txt
42
+ # "train: 1.0 0:1 /scratch/project_462000119/data/oscar_subsampled/gpt2tok_oscar_en_55B_text_document"
43
+ VALID_DATA_PATH=valc4oscar.txt
44
+ # "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
45
+
46
+ PP_SIZE=1
47
+ TP_SIZE=1
48
+
49
+ MICRO_BATCH_SIZE=2
50
+ GRADIENT_ACCUMULATION_STEPS=1
51
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
52
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
53
+
54
+ # Model parameters
55
+ source model_params.sh
56
+ MODEL_PARAM=("${PARAM_2980M[@]}")
57
+ NHIDDEN=${MODEL_PARAM[0]}
58
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
59
+ KV_SIZE=${MODEL_PARAM[2]}
60
+ NHEADS=${MODEL_PARAM[3]}
61
+ NLAYERS=${MODEL_PARAM[4]}
62
+ SEQ_LEN=2048
63
+
64
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
65
+
66
+ SAVE_INTERVAL=10000
67
+
68
+ # Tokens: 55000000000
69
+ # -> Samples: 26855469
70
+ TRAIN_SAMPLES=26_855_469
71
+
72
+ OPTIMIZER_ARGS=" \
73
+ --optimizer adam \
74
+ --adam-beta1 0.9 \
75
+ --adam-beta2 0.95 \
76
+ --adam-eps 1e-8 \
77
+ --lr 2e-4 \
78
+ --min-lr 2e-5 \
79
+ --lr-decay-style cosine \
80
+ --lr-decay-samples $TRAIN_SAMPLES \
81
+ --lr-warmup-samples 268_555 \
82
+ --clip-grad 1.0 \
83
+ --weight-decay 1e-1 \
84
+ "
85
+
86
+ GPT_ARGS=" \
87
+ --num-layers $NLAYERS \
88
+ --hidden-size $NHIDDEN \
89
+ --num-attention-heads $NHEADS \
90
+ --kv-channels $KV_SIZE \
91
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
92
+ --seq-length $SEQ_LEN \
93
+ --max-position-embeddings $SEQ_LEN \
94
+ --micro-batch-size $MICRO_BATCH_SIZE \
95
+ --global-batch-size $GLOBAL_BATCH_SIZE \
96
+ --train-samples $TRAIN_SAMPLES \
97
+ --vocab-file $VOCAB_FILE \
98
+ --merge-file $MERGE_FILE \
99
+ --clip-grad 1.0 \
100
+ --kill-switch-path $KILL_SWITCH_PATH \
101
+ --bf16 \
102
+ $OPTIMIZER_ARGS \
103
+ "
104
+
105
+ OUTPUT_ARGS=" \
106
+ --log-interval 10 \
107
+ --save-interval $SAVE_INTERVAL \
108
+ --eval-interval 100 \
109
+ --eval-iters 100 \
110
+ --tensorboard-dir $TENSORBOARD_PATH \
111
+ --tensorboard-queue-size 5 \
112
+ --log-timers-to-tensorboard \
113
+ --log-batch-size-to-tensorboard \
114
+ --log-validation-ppl-to-tensorboard \
115
+ "
116
+
117
+ ZERO_STAGE=0
118
+
119
+ mkdir -p ds_configs
120
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
121
+
122
+ cat <<EOF > $DS_CONFIG_PATH
123
+ {
124
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
125
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
126
+ "gradient_clipping": 1.0,
127
+ "zero_optimization": {
128
+ "stage": $ZERO_STAGE
129
+ },
130
+ "bf16": {
131
+ "enabled": true
132
+ },
133
+ "steps_per_print": 2000,
134
+ "wall_clock_breakdown": false
135
+ }
136
+ EOF
137
+
138
+ DEEPSPEED_ARGS=" \
139
+ --deepspeed \
140
+ --deepspeed_config $DS_CONFIG_PATH \
141
+ --zero-stage $ZERO_STAGE \
142
+ "
143
+
144
+ CMD=" \
145
+ Megatron-DeepSpeed/pretrain_gpt.py \
146
+ --tensor-model-parallel-size $TP_SIZE \
147
+ --pipeline-model-parallel-size $PP_SIZE \
148
+ $GPT_ARGS \
149
+ $OUTPUT_ARGS \
150
+ --save $CHECKPOINT_PATH \
151
+ --load $CHECKPOINT_PATH \
152
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
153
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
154
+ --data-impl mmap \
155
+ --seed 2 \
156
+ $DEEPSPEED_ARGS \
157
+ "
158
+
159
+ echo $CMD
160
+
161
+ echo "START $SLURM_JOBID: $(date)"
162
+
163
+ # bash launch_srun.sh $CMD
164
+ srun --label launch.sh $CMD
165
+
166
+ echo "END $SLURM_JOBID: $(date)"