diff --git a/train.py b/train.py index 19ad54a..8cbfe19 100644 --- a/train.py +++ b/train.py @@ -168,20 +168,8 @@ def train(config_path): eval_strategy=config.get("eval_strategy", "steps"), eval_steps=config.get("eval_steps", 100), bf16=True, - # gradient_checkpointing removed - FSDP activation_checkpointing handles it - fsdp=True, - fsdp_config={ - "sharding_strategy": "SHARD_GRAD_OP", - "cpu_offload": False, - "activation_checkpointing": True, - "limit_all_gathers": True, - "sync_module_states": False, # Don't sync from CPU, FSDP will handle placement - "backward_prefetch": "backward_pre", - "forward_prefetch": "true", - "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", - "transformer_layer_cls_to_wrap": "Qwen3_5MoeDecoderLayer", - "mixed_precision": {"param_dtype": torch.bfloat16, "reduce_dtype": torch.float32, "buffer_dtype": torch.float32}, - }, + gradient_checkpointing=True, + # No FSDP - use standard accelerate data parallelism ) # SFT Trainer (DeepSpeed handles distributed training via config)