cmedina/agenx-lora-training
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: force FSDP1 with string value to avoid FSDP2 gather spike

Browse Source
This commit is contained in:
Christian Medina
2026-07-02 10:54:17 -04:00
parent f1e016c8af
commit b056ec0306
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
train.py
View File

@@ -169,7 +169,7 @@ def train(config_path):
eval_steps=config.get("eval_steps", 100), eval_steps=config.get("eval_steps", 100),
bf16=True, bf16=True,
gradient_checkpointing=False, # FSDP activation_checkpointing handles it gradient_checkpointing=False, # FSDP activation_checkpointing handles it
fsdp=True, fsdp="full_shard", # Force FSDP1 (not FSDP2)
fsdp_config={ fsdp_config={
"sharding_strategy": "SHARD_GRAD_OP", "sharding_strategy": "SHARD_GRAD_OP",
"cpu_offload": False, "cpu_offload": False,