feat: test BnB 4-bit loading and GPU transfer

check_model_size.py

@@ -1,22 +1,34 @@
 #!/usr/bin/env python3
-"""Check model size and quantization status."""
+"""Test loading bf16 model with BnB 4-bit to CPU, then report size."""
 
 import torch
-from transformers import AutoModelForCausalLM
+from transformers import AutoModelForCausalLM, BitsAndBytesConfig
 
-model_path = "/data/models/Ornith-1.0-35B-bnb-4bit"
+model_path = "/data/models/Ornith-1.0-35B"
 
 print(f"Loading model from: {model_path}")
+print(f"\nApplying BnB 4-bit quantization...")
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.float16,
+)
+
 model = AutoModelForCausalLM.from_pretrained(
     model_path,
+    quantization_config=bnb_config,
     device_map="cpu",
     torch_dtype=torch.float16,
     trust_remote_code=True,
+    low_cpu_mem_usage=True,
 )
 
+print("✓ Model loaded to CPU with BnB 4-bit")
+
 # Count parameters
 total_params = sum(p.numel() for p in model.parameters())
-print(f"Total parameters: {total_params / 1e9:.2f}B")
+print(f"\nTotal parameters: {total_params / 1e9:.2f}B")
 
 # Check for quantized parameters
 bnb_params = 0
@@ -31,6 +43,16 @@ print(f"BnB 4-bit parameters: {bnb_params / 1e9:.2f}B")
 print(f"BF16 parameters: {bf16_params / 1e9:.2f}B")
 print(f"Estimated size: {(bnb_params * 0.5 + bf16_params * 2) / 1e9:.2f} GB")
 
+# Try to move to GPU
+print("\n   Moving to GPU 0...")
+try:
+    model = model.to("cuda:0")
+    print(f"✓ Success! GPU 0: {torch.cuda.memory_allocated(0) / 1e9:.2f} GB")
+    print(f"   Free VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9 - torch.cuda.memory_allocated(0) / 1e9:.2f} GB")
+except Exception as e:
+    print(f"✗ FAILED: {e}")
+
 del model
 import gc
 gc.collect()
+torch.cuda.empty_cache()