fix: disable FSDP, use standard accelerate data parallelism
This commit is contained in:
16
train.py
16
train.py
@@ -168,20 +168,8 @@ def train(config_path):
|
||||
eval_strategy=config.get("eval_strategy", "steps"),
|
||||
eval_steps=config.get("eval_steps", 100),
|
||||
bf16=True,
|
||||
# gradient_checkpointing removed - FSDP activation_checkpointing handles it
|
||||
fsdp=True,
|
||||
fsdp_config={
|
||||
"sharding_strategy": "SHARD_GRAD_OP",
|
||||
"cpu_offload": False,
|
||||
"activation_checkpointing": True,
|
||||
"limit_all_gathers": True,
|
||||
"sync_module_states": False, # Don't sync from CPU, FSDP will handle placement
|
||||
"backward_prefetch": "backward_pre",
|
||||
"forward_prefetch": "true",
|
||||
"auto_wrap_policy": "TRANSFORMER_BASED_WRAP",
|
||||
"transformer_layer_cls_to_wrap": "Qwen3_5MoeDecoderLayer",
|
||||
"mixed_precision": {"param_dtype": torch.bfloat16, "reduce_dtype": torch.float32, "buffer_dtype": torch.float32},
|
||||
},
|
||||
gradient_checkpointing=True,
|
||||
# No FSDP - use standard accelerate data parallelism
|
||||
)
|
||||
|
||||
# SFT Trainer (DeepSpeed handles distributed training via config)
|
||||
|
||||
Reference in New Issue
Block a user