cmedina/agenx-lora-training
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: reduce seq_length to 512 and increase gradient_accumulation

Browse Source
This commit is contained in:
Christian Medina
2026-07-02 17:53:11 -04:00
parent 691e5d3611
commit a51a744d7c
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
training/configs/ornith-35b-lora.yaml
View File

@@ -32,12 +32,12 @@ dataset:
train_params: train_params:
num_train_epochs: 3 num_train_epochs: 3
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 16 # Increased to reduce per-step memory
learning_rate: 0.0002 learning_rate: 0.0002
lr_scheduler_type: cosine lr_scheduler_type: cosine
weight_decay: 0.01 weight_decay: 0.01
warmup_ratio: 0.03 # Will be converted to warmup_steps by TrainingArguments warmup_ratio: 0.03 # Will be converted to warmup_steps by TrainingArguments
max_seq_length: 1024 max_seq_length: 512 # Reduced from 1024 to save memory
logging_steps: 10 logging_steps: 10
save_steps: 100 save_steps: 100
save_total_limit: 3 save_total_limit: 3