fix steps check for anneal on first cycle (#1316)
Browse files
src/axolotl/monkeypatch/relora.py
CHANGED
@@ -267,7 +267,7 @@ class ReLoRAScheduler(LRScheduler):
|
|
267 |
original = self.inner_schedule.get_lr()
|
268 |
step = self.last_epoch
|
269 |
|
270 |
-
if step < self.relora_steps:
|
271 |
scale = 1
|
272 |
else:
|
273 |
per_relora_progress = step % self.relora_steps
|
|
|
267 |
original = self.inner_schedule.get_lr()
|
268 |
step = self.last_epoch
|
269 |
|
270 |
+
if step < self.relora_steps - self.warmup_steps:
|
271 |
scale = 1
|
272 |
else:
|
273 |
per_relora_progress = step % self.relora_steps
|