feat: load bf16 model, quantize with BnB on CPU, move to GPU

train.py

@@ -33,22 +33,49 @@ def train(config_path):
     
     print(f"Loading model: {config['base_model']}")
     
-    # Load 4-bit model directly (already quantized on disk)
+    # Load bf16 model to CPU, then quantize with BnB
-    print(f"\n[INFO] Loading {config['base_model']} (4-bit)...")
+    print(f"\n[INFO] Loading {config['base_model']} bf16 to CPU...")
     model = AutoModelForCausalLM.from_pretrained(
         config["base_model"],
-        device_map="auto",  # Let transformers distribute
+        device_map="cpu",
+        torch_dtype=torch.bfloat16,
+        trust_remote_code=True,
+        low_cpu_mem_usage=True,
+    )
+    print("✓ Model loaded to CPU (~70GB bf16)")
+    
+    # Apply BnB 4-bit quantization on CPU
+    print("   Applying BnB 4-bit quantization on CPU...")
+    from transformers import BitsAndBytesConfig
+    bnb_config = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_compute_dtype=torch.float16,
+        bnb_4bit_use_double_quant=True,
+    )
+    
+    # Reload with quantization config
+    del model
+    import gc
+    gc.collect()
+    torch.cuda.empty_cache()
+    
+    model = AutoModelForCausalLM.from_pretrained(
+        config["base_model"],
+        device_map="cpu",
+        quantization_config=bnb_config,
         torch_dtype=torch.float16,
         trust_remote_code=True,
         low_cpu_mem_usage=True,
     )
-    print("✓ Success: Model loaded")
+    print("✓ Model quantized to 4-bit on CPU (~17.5GB)")
     
-    # Check memory usage
+    # Move to GPU
-    for i in range(torch.cuda.device_count()):
+    print("   Moving to GPU 0...")
-        mem = torch.cuda.memory_allocated(i) / 1e9
+    model = model.to("cuda:0")
-        total = torch.cuda.get_device_properties(i).total_memory / 1e9
+    print("✓ Success: Model loaded to GPU 0 (4-bit)")
-        print(f"   GPU {i}: {mem:.2f} GB / {total:.2f} GB")
+    print(f"   GPU 0: {torch.cuda.memory_allocated(0) / 1e9:.2f} GB")
+    print(f"   Free VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9 - torch.cuda.memory_allocated(0) / 1e9:.2f} GB")
     
     # Add LoRA
     lora_config = LoraConfig(

training/configs/ornith-35b-lora.yaml

@@ -1,7 +1,7 @@
 # LoRA Training Configuration for Ornith-1.0-35B
 # Dataset: cyron_summary_lora_dataset (20k examples)
 
-base_model: /data/models/Ornith-1.0-35B-4bit
+base_model: /data/models/Ornith-1.0-35B
 model_type: Qwen3_5MoeForCausalLM
 tokenizer_type: AutoTokenizer