Hugging Face's logo

distily
/
distily_test_attn_miles

TensorBoard
Safetensors
Distily
gpt2
bitnet
1.58b
Generated from Trainer
Model card Files Metrics Community
lapp0 commited on
Commit
6694218
·
verified ·
1 Parent(s): 440d7bd

End of training

Browse files
Files changed (47) hide show
  1. .gitattributes +1 -0
  2. README.md +156 -0
  3. benchmarks.shelve.bak +2 -0
  4. benchmarks.shelve.dat +3 -0
  5. benchmarks.shelve.dir +2 -0
  6. config.json +39 -0
  7. generation_config.json +6 -0
  8. logs/attn_layer_mapper=all, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724529981.f383272e719b +3 -0
  9. logs/attn_layer_mapper=all, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724530058.f383272e719b +3 -0
  10. logs/attn_layer_mapper=all, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724531531.f383272e719b +3 -0
  11. logs/attn_layer_mapper=all, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724531610.f383272e719b +3 -0
  12. logs/attn_layer_mapper=all, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529252.f383272e719b +3 -0
  13. logs/attn_layer_mapper=all, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529446.f383272e719b +3 -0
  14. logs/attn_layer_mapper=all, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724530885.f383272e719b +3 -0
  15. logs/attn_layer_mapper=all, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724530975.f383272e719b +3 -0
  16. logs/attn_layer_mapper=all, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724532492.f383272e719b +3 -0
  17. logs/attn_layer_mapper=last, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724530286.f383272e719b +3 -0
  18. logs/attn_layer_mapper=last, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724530365.f383272e719b +3 -0
  19. logs/attn_layer_mapper=last, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724531845.f383272e719b +3 -0
  20. logs/attn_layer_mapper=last, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724531923.f383272e719b +3 -0
  21. logs/attn_layer_mapper=last, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529682.f383272e719b +3 -0
  22. logs/attn_layer_mapper=last, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529759.f383272e719b +3 -0
  23. logs/attn_layer_mapper=last, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724531213.f383272e719b +3 -0
  24. logs/attn_layer_mapper=last, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724531290.f383272e719b +3 -0
  25. logs/attn_layer_mapper=last_k_2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724530133.f383272e719b +3 -0
  26. logs/attn_layer_mapper=last_k_2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724530208.f383272e719b +3 -0
  27. logs/attn_layer_mapper=last_k_2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724531689.f383272e719b +3 -0
  28. logs/attn_layer_mapper=last_k_2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724531765.f383272e719b +3 -0
  29. logs/attn_layer_mapper=last_k_2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529528.f383272e719b +3 -0
  30. logs/attn_layer_mapper=last_k_2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529604.f383272e719b +3 -0
  31. logs/attn_layer_mapper=last_k_2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724531057.f383272e719b +3 -0
  32. logs/attn_layer_mapper=last_k_2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724531138.f383272e719b +3 -0
  33. logs/attn_layer_mapper=layer-2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724530444.f383272e719b +3 -0
  34. logs/attn_layer_mapper=layer-2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724530520.f383272e719b +3 -0
  35. logs/attn_layer_mapper=layer-2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724532004.f383272e719b +3 -0
  36. logs/attn_layer_mapper=layer-2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724532080.f383272e719b +3 -0
  37. logs/attn_layer_mapper=layer-2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529831.f383272e719b +3 -0
  38. logs/attn_layer_mapper=layer-2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529907.f383272e719b +3 -0
  39. logs/attn_layer_mapper=layer-2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724531375.f383272e719b +3 -0
  40. logs/attn_layer_mapper=layer-2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724531452.f383272e719b +3 -0
  41. merges.txt +0 -0
  42. model.safetensors +3 -0
  43. special_tokens_map.json +6 -0
  44. tokenizer.json +0 -0
  45. tokenizer_config.json +20 -0
  46. training_args.bin +3 -0
  47. vocab.json +0 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ benchmarks.shelve.dat filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: gpt2
3
+ datasets:
4
+ - wikimedia/wikipedia
5
+ library_name: Distily
6
+ license: mit
7
+ tags:
8
+ - bitnet
9
+ - 1.58b
10
+ - generated_from_trainer
11
+ model-index:
12
+ - name: distily_test_attn_miles
13
+ results: []
14
+ ---
15
+
16
+
17
+ # Summary
18
+
19
+ Distilled with [Distily](https://github.com/lapp0/distily) library
20
+ using teacher model [gpt2](https://huggingface.co/gpt2)
21
+ on dataset [wikimedia/wikipedia](https://huggingface.co/datasets/wikimedia/wikipedia).
22
+
23
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
24
+ should probably proofread and complete it, then remove this comment.
25
+
26
+ # Model description
27
+
28
+ More information needed
29
+
30
+ # Intended uses & limitations
31
+
32
+ More information needed
33
+ -->
34
+
35
+ # Model Architecture:
36
+ - **Architecture**: `GPT2LMHeadModel`
37
+ - **Total Parameters**: 124,439,808
38
+ - **Data Type (dtype)**: torch.bfloat16
39
+ - **Model Size**: 0.24 GB
40
+
41
+
42
+ # Benchmark Metrics Comparison
43
+
44
+ | Metric | attn_layer_mapper=all, attn_loss_fn=raw_mse, attn_projector=miles | teacher |
45
+ | :--- | :--- | :--- |
46
+ | ai2_arc (acc) | 0.256 | 0.304 |
47
+ | ai2_arc (acc_norm) | 0.267 | 0.309 |
48
+ | arc_challenge (acc) | 0.177 | 0.184 |
49
+ | arc_challenge (acc_norm) | 0.202 | 0.214 |
50
+ | arc_easy (acc) | 0.335 | 0.424 |
51
+ | arc_easy (acc_norm) | 0.332 | 0.405 |
52
+ | boolq (acc) | 0.377 | 0.541 |
53
+ | cola (mcc) | 0.0 | 0.009 |
54
+ | glue (acc) | 0.444 | 0.41 |
55
+ | glue (f1) | 0.279 | 0.526 |
56
+ | glue (mcc) | 0.0 | 0.009 |
57
+ | hellaswag (acc) | 0.302 | 0.337 |
58
+ | hellaswag (acc_norm) | 0.308 | 0.384 |
59
+ | mnli (acc) | 0.331 | 0.323 |
60
+ | mnli_mismatch (acc) | 0.367 | 0.344 |
61
+ | mrpc (acc) | 0.336 | 0.515 |
62
+ | mrpc (f1) | 0.075 | 0.631 |
63
+ | qnli (acc) | 0.519 | 0.472 |
64
+ | qqp (acc) | 0.515 | 0.34 |
65
+ | qqp (f1) | 0.363 | 0.483 |
66
+ | rte (acc) | 0.57 | 0.516 |
67
+ | sst2 (acc) | 0.498 | 0.511 |
68
+ | wikitext (bits_per_byte) | 1.273 | 0.98 |
69
+ | wikitext (byte_perplexity) | 2.416 | 1.973 |
70
+ | wikitext (word_perplexity) | 111.9 | 37.82 |
71
+ | wnli (acc) | 0.521 | 0.451 |
72
+
73
+ # Resource Usage Comparison
74
+
75
+ - VRAM Use: 7.7854 GB
76
+
77
+ # Distillation (Teacher -> Student) Architecture Difference:
78
+
79
+ - **Architecture**: `GPT2LMHeadModel` -> `GPT2LMHeadModel`
80
+ - **Total Parameters**: 124,439,808 -> 124,439,808
81
+ - **Data Type (dtype)**: torch.bfloat16 -> torch.bfloat16
82
+ - **Model Size**: 0.24 GB -> 0.24 GB
83
+
84
+ <details>
85
+ <summary>Module Diff Details</summary>
86
+
87
+ ```diff
88
+
89
+ ```
90
+
91
+ </details>
92
+ <br/>
93
+
94
+ # Train Dataset
95
+ Trained on 145,722,156 tokens from the [wikimedia/wikipedia](https://huggingface.co/datasets/wikimedia/wikipedia) dataset.
96
+
97
+ - Num Samples: `247,500`
98
+ - Subset: `20231101.en`
99
+ - Split: `train`
100
+
101
+
102
+ # Training Objective
103
+
104
+ ```
105
+ DistillationObjective(logits_loss_component=LossComponent(label=logits, weight=1, loss_fn=kl), attn_loss_component=LossComponent(label=attn, weight=25.0, loss_fn=raw_mse, layer_mapper=all, projector=miles))
106
+ ```
107
+
108
+ # Hyperparameters
109
+ The following hyperparameters were used during training:
110
+
111
+ <details>
112
+ <summary>Expand</summary>
113
+
114
+ - learning_rate: `0.0001`
115
+ - train_batch_size: `4`
116
+ - eval_batch_size: `8`
117
+ - seed: `42`
118
+ - optimizer: `Adam with betas=(0.9,0.999) and epsilon=1e-08`
119
+ - lr_scheduler_type: `cosine_with_min_lr`
120
+ - lr_scheduler_warmup_ratio: `0.5`
121
+ - num_epochs: `1.0`
122
+ - distillation_objective: `DistillationObjective(logits_loss_component=LossComponent(label=logits, weight=1, loss_fn=kl), attn_loss_component=LossComponent(label=attn, weight=25.0, loss_fn=raw_mse, layer_mapper=all, projector=miles))`
123
+ - train_embeddings: `True`
124
+ - lr_scheduler: `<torch.optim.lr_scheduler.LambdaLR object at 0x7f6efc464fd0>`
125
+ - student_model_name_or_path: `None`
126
+ - student_config_name_or_path: `None`
127
+ - student_model_config: `None`
128
+ - reinitialize_weights: `None`
129
+ - copy_teacher_modules: `[('lm_head', False)]`
130
+ - student_model_as_bitnet: `True`
131
+ - dropout: `None`
132
+ - teacher_model_name_or_path: `gpt2`
133
+ - teacher_load_in_8bit: `False`
134
+ - teacher_load_in_4bit: `False`
135
+ - dataset_uri: `wikimedia/wikipedia`
136
+ - dataset_subset: `20231101.en`
137
+ - dataset_split: `train`
138
+ - dataset_column_name: `text`
139
+ - dataset_sample_size: `250000`
140
+ - dataset_test_size: `0.01`
141
+ - gradient_accumulation_steps: `1`
142
+ - weight_decay: `0.0`
143
+ - max_grad_norm: `1.0`
144
+ - warmup_ratio: `0.5`
145
+ - warmup_steps: `0`
146
+ - gradient_checkpointing: `True`
147
+
148
+ </details>
149
+ <br/>
150
+
151
+
152
+ # Framework Versions
153
+ - Distily 0.3.0
154
+ - Transformers 4.44.0
155
+ - Pytorch 2.3.0
156
+ - Datasets 2.21.0
benchmarks.shelve.bak ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ 'teacher', (0, 14412556)
2
+ 'attn_layer_mapper=all, attn_loss_fn=raw_mse, attn_projector=miles', (14412800, 14412543)
benchmarks.shelve.dat ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:15752df966654d7744eed260668d6b9896db6cad1636bbc6ea69f57a26dbf0c1
3
+ size 28825343
benchmarks.shelve.dir ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ 'teacher', (0, 14412556)
2
+ 'attn_layer_mapper=all, attn_loss_fn=raw_mse, attn_projector=miles', (14412800, 14412543)
config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "gpt2",
3
+ "activation_function": "gelu_new",
4
+ "architectures": [
5
+ "GPT2LMHeadModel"
6
+ ],
7
+ "attn_pdrop": 0.1,
8
+ "bos_token_id": 50256,
9
+ "embd_pdrop": 0.1,
10
+ "eos_token_id": 50256,
11
+ "initializer_range": 0.02,
12
+ "layer_norm_epsilon": 1e-05,
13
+ "model_type": "gpt2",
14
+ "n_ctx": 1024,
15
+ "n_embd": 768,
16
+ "n_head": 12,
17
+ "n_inner": null,
18
+ "n_layer": 12,
19
+ "n_positions": 1024,
20
+ "reorder_and_upcast_attn": false,
21
+ "resid_pdrop": 0.1,
22
+ "scale_attn_by_inverse_layer_idx": false,
23
+ "scale_attn_weights": true,
24
+ "summary_activation": null,
25
+ "summary_first_dropout": 0.1,
26
+ "summary_proj_to_labels": true,
27
+ "summary_type": "cls_index",
28
+ "summary_use_proj": true,
29
+ "task_specific_params": {
30
+ "text-generation": {
31
+ "do_sample": true,
32
+ "max_length": 50
33
+ }
34
+ },
35
+ "torch_dtype": "bfloat16",
36
+ "transformers_version": "4.44.0",
37
+ "use_cache": true,
38
+ "vocab_size": 50257
39
+ }
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 50256,
4
+ "eos_token_id": 50256,
5
+ "transformers_version": "4.44.0"
6
+ }
logs/attn_layer_mapper=all, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724529981.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:16af67e1c7469a0965a7e9d6ebd2275a2d50d7858441e6b0d953240642ccf871
3
+ size 5553
logs/attn_layer_mapper=all, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724530058.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8c4c1668c6dd8d17c4d2af3067e4a5c1f62a2507432ee180112ba394b80af1b
3
+ size 5553
logs/attn_layer_mapper=all, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724531531.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ea4c9bcc17d6390ed9d7f2f9836cee6aa100c5f5d37faf9c373fc1a30a2b9449
3
+ size 5553
logs/attn_layer_mapper=all, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724531610.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2f33bfb85ac56b5e188389d1e7687844b4840e7ec787b8acdbf729fba39821f5
3
+ size 5553
logs/attn_layer_mapper=all, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529252.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7a34b8a2fccadd9def77e9f7df080880cff1d7dd2d20fcabb524b3e0a513ac86
3
+ size 5561
logs/attn_layer_mapper=all, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529446.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:574c48352958c8dd8a9c71fccd47f0a1c493f8d653287820af2abfb3d2aa6492
3
+ size 5561
logs/attn_layer_mapper=all, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724530885.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc32ff84f307ac76c6e4cb58cc66fa5ba04fb3e5c84a9ded78e2bc560aea3a82
3
+ size 5561
logs/attn_layer_mapper=all, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724530975.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:33a2e5c0a157747ec792dd0ca7301e3f5a17a0e249d4b40319891b300763e81d
3
+ size 5561
logs/attn_layer_mapper=all, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724532492.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c20f14135e4a0934ae1fa21f40c9158c96c946258aec265af93361af9100865b
3
+ size 29625283
logs/attn_layer_mapper=last, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724530286.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44fc7b48f78cc25a6bfdd66b66ce154115b985128636a6d454f800a72a5b2326
3
+ size 5555
logs/attn_layer_mapper=last, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724530365.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:71bd81346b9182f638fd752b81b1fd4b6a4691c841705b3252502d617cccba67
3
+ size 5555
logs/attn_layer_mapper=last, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724531845.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d5d1bc1c2a83a3680f9590ce879dc1fb9893de51edce149f90099ce9d36c680
3
+ size 5555
logs/attn_layer_mapper=last, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724531923.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a3d746b1ffdc5c628ca8995814834638eef11820d103efd4efbc62deab1add29
3
+ size 5555
logs/attn_layer_mapper=last, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529682.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f2b707bb7295fe7f5940d7b3cba950c031e727c7812aaa188a2cc234ccad9fcf
3
+ size 5563
logs/attn_layer_mapper=last, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529759.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3453e0a30045487db8b012bfadb3f263eae531d79cd0a79078b1a0c2dda7c5ba
3
+ size 5563
logs/attn_layer_mapper=last, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724531213.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fd55a541b0ea4b0f87d932b4c5784a7d2010f2d6d20c0855f3442fe19214c161
3
+ size 5563
logs/attn_layer_mapper=last, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724531290.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:45c19a17231e6e4a505b69d19d961c2d499ca23f174d1c17eb2006b8961fd65e
3
+ size 5563
logs/attn_layer_mapper=last_k_2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724530133.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3953540e07dec03f0e9437760b7c77962cc0420724c74be0d6b15c547bdc0727
3
+ size 5563
logs/attn_layer_mapper=last_k_2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724530208.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3a10db52e85d921b839bdd0e10442fbf9ea08b6698f3a862d475bd59417511f6
3
+ size 5563
logs/attn_layer_mapper=last_k_2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724531689.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51b28b09ba3afbbf524b0dab8fc0edd34c9b64c25017736c8df7d1e8eba5d7fe
3
+ size 5563
logs/attn_layer_mapper=last_k_2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724531765.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44086ba7ed99db685c97f4d7c21042c9b69df212f5efb5900eaae7de1a7ca1c0
3
+ size 5563
logs/attn_layer_mapper=last_k_2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529528.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c4c0aed60395eae9aa7d8e965b035f7b905053c7e1cc10dacb43196b6b41c84
3
+ size 5571
logs/attn_layer_mapper=last_k_2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529604.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:748d01a1dde4dfa3d58c395577224c58ca3c7cb5418a98e58276af7e1326402c
3
+ size 5571
logs/attn_layer_mapper=last_k_2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724531057.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9cfaac7c06de9b5e1ec5faa714418c66ff4bbaf7e306a6da6fa4171df2ad8f7
3
+ size 5571
logs/attn_layer_mapper=last_k_2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724531138.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:16c0b129b9f457ac55b8fca033011ac921adcb0af89f9bb4f3d14935fcd94206
3
+ size 5571
logs/attn_layer_mapper=layer-2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724530444.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1a9cfaa4c7087bc576f30ef01ac47384b43a1f09543b8cdff957747d0e84f78f
3
+ size 5561
logs/attn_layer_mapper=layer-2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724530520.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9fc5116eb2b571934284b4034ed7a1ac5b2e5b319625a9245c615919790d5add
3
+ size 5561
logs/attn_layer_mapper=layer-2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724532004.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3c6a8be30f10a837b5d08fef69f02df1d32cdd336a0eb437c4a9887d81f575d
3
+ size 5561
logs/attn_layer_mapper=layer-2, attn_loss_fn=cos, attn_projector=miles/events.out.tfevents.1724532080.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:014ffb779da5d7d10198cd62d27cbae0693ffcd54e8e66b8a7d2cc3770ecfe51
3
+ size 5561
logs/attn_layer_mapper=layer-2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529831.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:abe47c463f082c747f4aff5f606b8ef29b2418baa2209efed01232109a1b0c4f
3
+ size 5569
logs/attn_layer_mapper=layer-2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724529907.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0f9227e9e245a62ae41831a245143a330ba685e251a603414a1d8ac39521438e
3
+ size 5569
logs/attn_layer_mapper=layer-2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724531375.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:67683225f5ef42d3af2a3565bdec9adf11b1d9b921bbbbf36773ae1cce7ae622
3
+ size 5569
logs/attn_layer_mapper=layer-2, attn_loss_fn=raw_mse, attn_projector=miles/events.out.tfevents.1724531452.f383272e719b ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:082f659e62aea7eb6898ffce83c8827cdb2eda67399921ad0e63673e3ddf690a
3
+ size 5569
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e4e12deaeab134bd4bfa52f1550a57b8d68ded99389617b8a73665515c24aed
3
+ size 248894656
special_tokens_map.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "pad_token": "<|endoftext|>",
5
+ "unk_token": "<|endoftext|>"
6
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "50256": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ }
12
+ },
13
+ "bos_token": "<|endoftext|>",
14
+ "clean_up_tokenization_spaces": true,
15
+ "eos_token": "<|endoftext|>",
16
+ "model_max_length": 1024,
17
+ "pad_token": "<|endoftext|>",
18
+ "tokenizer_class": "GPT2Tokenizer",
19
+ "unk_token": "<|endoftext|>"
20
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:69e8f4d6c8a98709c31aaccd719d69d1f82ff7f7d0023a62fe646d14320a8512
3
+ size 5432
vocab.json ADDED
The diff for this file is too large to render. See raw diff