diff --git a/training/configs/ornith-35b-lora.yaml b/training/configs/ornith-35b-lora.yaml
index 06e9f13..22b8583 100644
--- a/training/configs/ornith-35b-lora.yaml
+++ b/training/configs/ornith-35b-lora.yaml
@@ -53,6 +53,12 @@ huggingface_hub:
 deepspeed_config:
   zero_optimization:
     stage: 3
+    offload_optimizer:
+      device: cpu
+      pin_memory: true
+    offload_param:
+      device: cpu
+      pin_memory: true
   gradient_clipping: 1.0
   train_batch_size: auto
   train_micro_batch_size_per_gpu: auto