fix: skip quantization, use DeepSpeed ZeRO-3 CPU offload directly

training/scripts/train.py

@@ -36,56 +36,15 @@ def train(config_path):
     print(f"Loading model: {config['base_model']}")
     from transformers import BitsAndBytesConfig, AutoConfig
     
-    try:
+    # Skip quantization - use DeepSpeed ZeRO-3 with CPU offload
-        # Try 4-bit QLoRA first
+    print(f"Loading model: {config['base_model']}")
-        bnb_config = BitsAndBytesConfig(
+    model = AutoModelForCausalLM.from_pretrained(
-            load_in_4bit=True,
+        config["base_model"],
-            bnb_4bit_quant_type="nf4",
+        torch_dtype=torch.bfloat16,
-            bnb_4bit_compute_dtype=torch.bfloat16,
+        device_map="cpu",  # Load to CPU first
-            bnb_4bit_use_double_quant=True,
+        trust_remote_code=True,
-        )
+    )
-        model = AutoModelForCausalLM.from_pretrained(
+    print("Model loaded to CPU. DeepSpeed will distribute.")
-            config["base_model"],
-            quantization_config=bnb_config,
-            dtype=torch.bfloat16,
-            device_map="auto",
-            trust_remote_code=True,
-        )
-        print("Model loaded with QLoRA (4-bit).")
-    except Exception as e:
-        print(f"4-bit failed: {e}")
-        try:
-            # Try 8-bit with CPU offload
-            print("Trying 8-bit with CPU offload...")
-            bnb_config_8bit = BitsAndBytesConfig(
-                load_in_8bit=True,
-                llm_int8_enable_fp32_cpu_offload=True,
-            )
-            model = AutoModelForCausalLM.from_pretrained(
-                config["base_model"],
-                quantization_config=bnb_config_8bit,
-                device_map="auto",
-                trust_remote_code=True,
-            )
-            print("Model loaded with 8-bit CPU offload.")
-        except Exception as e2:
-            print(f"8-bit failed: {e2}, falling back to bf16 with accelerate CPU offload")
-            # Use accelerate to load with CPU offload
-            from accelerate import load_checkpoint_and_dispatch
-            model = AutoModelForCausalLM.from_pretrained(
-                config["base_model"],
-                torch_dtype=torch.bfloat16,
-                trust_remote_code=True,
-            )
-            # Load with CPU offload
-            model = load_checkpoint_and_dispatch(
-                model,
-                checkpoint=config["base_model"],
-                device_map="auto",
-                dtype=torch.bfloat16,
-                offload_folder="/tmp/model_offload",
-            )
-            print("Model loaded with accelerate CPU offload.")
     
     # Prepare model for k-bit training
     from peft import prepare_model_for_kbit_training