| adapter: lora | |
| base_model: NousResearch/Meta-Llama-3-8B-Alternate-Tokenizer | |
| batch_size: 64 | |
| bf16: true | |
| chat_template: tokenizer_default_fallback_alpaca | |
| datasets: | |
| - format: custom | |
| path: https://gradients.s3.eu-north-1.amazonaws.com/1ae63171fbf66651_train_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVVZOOA7SA4UOFLPI%2F20250210%2Feu-north-1%2Fs3%2Faws4_request&X-Amz-Date=20250210T120931Z&X-Amz-Expires=604800&X-Amz-SignedHeaders=host&X-Amz-Signature=bbe3efc2d3663c793088c33c43c09e6fbc67e548bccdd8400f529787d50520a6 | |
| type: | |
| field_input: headline_a | |
| field_instruction: rendered_input | |
| field_output: headline_b | |
| format: '{instruction} {input}' | |
| no_input_format: '{instruction}' | |
| system_format: '{system}' | |
| system_prompt: '' | |
| device_map: auto | |
| eval_sample_packing: false | |
| eval_steps: 0.1 | |
| flash_attention: true | |
| gradient_checkpointing: true | |
| group_by_length: true | |
| hub_model_id: SystemAdmin123/ce7a539a-b639-41f4-8114-4dc65059452a | |
| hub_strategy: checkpoint | |
| learning_rate: 0.0001 | |
| logging_steps: 10 | |
| lora_alpha: 256 | |
| lora_dropout: 0.1 | |
| lora_r: 128 | |
| lora_target_linear: true | |
| lr_scheduler: cosine | |
| max_steps: 8000 | |
| micro_batch_size: 1 | |
| model_type: AutoModelForCausalLM | |
| num_epochs: 10000 | |
| optimizer: adamw_bnb_8bit | |
| output_dir: /root/.sn56/axolotl/tmp/ce7a539a-b639-41f4-8114-4dc65059452a | |
| pad_to_sequence_len: true | |
| resize_token_embeddings_to_32x: false | |
| sample_packing: true | |
| save_steps: 40 | |
| save_total_limit: 1 | |
| sequence_len: 2048 | |
| tokenizer_type: PreTrainedTokenizerFast | |
| torch_dtype: bf16 | |
| training_args_kwargs: | |
| disable_tqdm: true | |
| hub_private_repo: true | |
| save_only_model: true | |
| trust_remote_code: true | |
| val_set_size: 0.01 | |
| wandb_entity: '' | |
| wandb_mode: online | |
| wandb_name: NousResearch/Meta-Llama-3-8B-Alternate-Tokenizer-https://gradients.s3.eu-north-1.amazonaws.com/1ae63171fbf66651_train_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVVZOOA7SA4UOFLPI%2F20250210%2Feu-north-1%2Fs3%2Faws4_request&X-Amz-Date=20250210T120931Z&X-Amz-Expires=604800&X-Amz-SignedHeaders=host&X-Amz-Signature=bbe3efc2d3663c793088c33c43c09e6fbc67e548bccdd8400f529787d50520a6 | |
| wandb_project: Gradients-On-Demand | |
| wandb_run: your_name | |
| wandb_runid: default | |
| warmup_ratio: 0.05 | |