| { | |
| "model_name": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", | |
| "model_type": "llama", | |
| "training_params": { | |
| "learning_rate": 2e-4, | |
| "num_train_epochs": 3, | |
| "per_device_train_batch_size": 4, | |
| "gradient_accumulation_steps": 4, | |
| "warmup_ratio": 0.03, | |
| "lr_scheduler_type": "cosine", | |
| "max_grad_norm": 0.3, | |
| "weight_decay": 0.01 | |
| }, | |
| "lora_config": { | |
| "r": 8, | |
| "lora_alpha": 16, | |
| "target_modules": ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"], | |
| "bias": "none", | |
| "task_type": "CAUSAL_LM" | |
| }, | |
| "generation_config": { | |
| "max_length": 200, | |
| "temperature": 0.7, | |
| "top_p": 0.9, | |
| "top_k": 40, | |
| "repetition_penalty": 1.1 | |
| }, | |
| "training_framework": { | |
| "name": "unsloth", | |
| "quantization": "4bit", | |
| "use_flash_attention": true, | |
| "use_peft": true | |
| } | |
| } | |