fix: add CPU offload back to DeepSpeed ZeRO-3

2026-06-30 22:47:19 -04:00
parent 3bb7328e2a
commit 5713bf0285
1 changed files with 6 additions and 0 deletions
--- a/training/configs/ornith-35b-lora.yaml
+++ b/training/configs/ornith-35b-lora.yaml
@@ -53,6 +53,12 @@ huggingface_hub:
 deepspeed_config:
  zero_optimization:
    stage: 3
+    offload_optimizer:
+      device: cpu
+      pin_memory: true
+    offload_param:
+      device: cpu
+      pin_memory: true
  gradient_clipping: 1.0
  train_batch_size: auto
  train_micro_batch_size_per_gpu: auto