feat: add prepare_model_for_kbit_training and update LoRA r=64 alpha=128

2026-07-01 07:48:44 -04:00
parent 357c9fa781
commit 2fa5173c56
2 changed files with 7 additions and 2 deletions
--- a/training/configs/ornith-35b-lora.yaml
+++ b/training/configs/ornith-35b-lora.yaml
@@ -9,8 +9,8 @@ tokenizer_type: LlamaTokenizer
 # No need for BitsAndBytes configuration

 # LoRA Configuration
-lora_r: 16
-lora_alpha: 32
+lora_r: 64
+lora_alpha: 128
 lora_dropout: 0.05
 target_modules:
  - q_proj