diff --git a/train.py b/train.py index a0d0a61..6d348ee 100644 --- a/train.py +++ b/train.py @@ -167,13 +167,14 @@ def train(config_path): eval_steps=config.get("eval_steps", 100), bf16=True, gradient_checkpointing=config.get("gradient_checkpointing", True), - fsdp=["full_shard", "auto_wrap"], + fsdp=True, fsdp_config={ "backward_prefetch": "backward_pre", "forward_prefetch": "true", "cpu_offload": "false", "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", "transformer_layer_cls_to_wrap": "Qwen3_5MoeDecoderLayer", + "activation_checkpointing": True, }, )