cmedina/agenx-lora-training
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: simplify FSDP config, comment out mixed_precision

Browse Source
This commit is contained in:
Christian Medina
2026-07-02 09:38:10 -04:00
parent ab1988705b
commit 361a58addc
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
train.py
View File

@@ -177,7 +177,7 @@ def train(config_path):
"limit_all_gathers": True, "limit_all_gathers": True,
"sync_module_states": True, "sync_module_states": True,
"use_orig_params": True, # Critical for LoRA/PEFT "use_orig_params": True, # Critical for LoRA/PEFT
"mixed_precision": {"param_dtype": torch.bfloat16, "reduce_dtype": torch.float32, "buffer_dtype": torch.float32}, # "mixed_precision": {"param_dtype": torch.bfloat16, "reduce_dtype": torch.float32, "buffer_dtype": torch.float32},
"auto_wrap_policy": "TRANSFORMER_BASED_WRAP", "auto_wrap_policy": "TRANSFORMER_BASED_WRAP",
"transformer_layer_cls_to_wrap": "Qwen3_5MoeDecoderLayer", "transformer_layer_cls_to_wrap": "Qwen3_5MoeDecoderLayer",
}, },