| activation: softmax | |
| adam_beta1: 0.9 | |
| adam_beta2: 0.99 | |
| adam_epsilon: 1.0e-06 | |
| alpha: 0.1 | |
| attn_implementation: null | |
| beta: 0.125 | |
| bf16: true | |
| block_size: 512 | |
| checkpoint_dir: mlruns/896390784617014591/892b97fa0aa6499288906c463545ae00/checkpoints | |
| compile: false | |
| config_path: configs/JZ/NRJ_base-wiki-original.yaml | |
| dataloader_num_workers: 8 | |
| dataset_path: /lustre/fswork/projects/rech/oou/uqh26ve/data/pre_training/en/en_wiki/wiki_20220301-cleaned-valid001/data-bin/wiki_20220301-cleaned-valid001-BPE30K/ | |
| ddp_find_unused_parameters: false | |
| disable_tqdm: true | |
| do_eval: true | |
| dropout: 0.1 | |
| embedding_dim: 768 | |
| eval_steps: 25000 | |
| evaluation_strategy: steps | |
| forward_memories: 3072 | |
| fp16: false | |
| gradient_accumulation_steps: 1 | |
| ignore_lines: false | |
| layer_norm: 1.0e-12 | |
| learning_rate: 0.0007 | |
| log_on_each_node: false | |
| logging_steps: 1000 | |
| logging_strategy: steps | |
| lr_scheduler_kwargs: {} | |
| lr_scheduler_type: cosine | |
| max_steps: 500000 | |
| model_name: NRJ-V_30000K_bpe-NL12-NH12-EMB768-FFN3072 | |
| model_type: energyBERT | |
| n_run: 51 | |
| num_heads: 12 | |
| num_layers: 12 | |
| num_params: 50638896 | |
| optimizer: adamw_torch | |
| output_dir: null | |
| per_device_eval_batch_size: 8 | |
| per_device_train_batch_size: 64 | |
| remove_unused_columns: false | |
| report_to: mlflow | |
| save_steps: 25000 | |
| save_strategy: steps | |
| seed: 42 | |
| share_layers: false | |
| test_file: /lustre/fswork/projects/rech/oou/uqh26ve/data/pre_training/en/en_wiki/wiki_20220301-cleaned-valid001/wikipedia.test.txt | |
| tie_weights: false | |
| tokenizer_path: /lustre/fswork/projects/rech/oou/uqh26ve/data/pre_training/en/en_wiki/wiki_20220301-cleaned-valid001/data-bin/wiki_20220301-cleaned-valid001-BPE30K/tokenizer | |
| tokenizer_type: bpe | |
| total_batch_size: 4096 | |
| training_file: /lustre/fswork/projects/rech/oou/uqh26ve/data/pre_training/en/en_wiki/wiki_20220301-cleaned-valid001/wikipedia.train.txt | |
| valid_file: /lustre/fswork/projects/rech/oou/uqh26ve/data/pre_training/en/en_wiki/wiki_20220301-cleaned-valid001/wikipedia.valid.txt | |
| vocabulary_size: 30000 | |
| warmup_ratio: 0.0 | |
| warmup_steps: 24000 | |
| weight_decay: 0.01 | |