cmedina/agenx-lora-training
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: add use_orig_params=True for LoRA/PEFT compatibility

Browse Source
This commit is contained in:
Christian Medina
2026-07-02 09:37:37 -04:00
parent b7d680966e
commit ab1988705b
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
train.py
View File

@@ -176,6 +176,7 @@ def train(config_path):
"activation_checkpointing": True, "activation_checkpointing": True,
"limit_all_gathers": True, "limit_all_gathers": True,
"sync_module_states": True, "sync_module_states": True,
"use_orig_params": True, # Critical for LoRA/PEFT
"mixed_precision": {"param_dtype": torch.bfloat16, "reduce_dtype": torch.float32, "buffer_dtype": torch.float32}, "mixed_precision": {"param_dtype": torch.bfloat16, "reduce_dtype": torch.float32, "buffer_dtype": torch.float32},
"auto_wrap_policy": "TRANSFORMER_BASED_WRAP", "auto_wrap_policy": "TRANSFORMER_BASED_WRAP",
"transformer_layer_cls_to_wrap": "Qwen3_5MoeDecoderLayer", "transformer_layer_cls_to_wrap": "Qwen3_5MoeDecoderLayer",