59 lines
1.6 KiB
Python
59 lines
1.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Test loading bf16 model with BnB 4-bit to CPU, then report size."""
|
|
|
|
import torch
|
|
from transformers import AutoModelForCausalLM, BitsAndBytesConfig
|
|
|
|
model_path = "/data/models/Ornith-1.0-35B"
|
|
|
|
print(f"Loading model from: {model_path}")
|
|
print(f"\nApplying BnB 4-bit quantization...")
|
|
|
|
bnb_config = BitsAndBytesConfig(
|
|
load_in_4bit=True,
|
|
bnb_4bit_quant_type="nf4",
|
|
bnb_4bit_compute_dtype=torch.float16,
|
|
)
|
|
|
|
model = AutoModelForCausalLM.from_pretrained(
|
|
model_path,
|
|
quantization_config=bnb_config,
|
|
device_map="cpu",
|
|
torch_dtype=torch.float16,
|
|
trust_remote_code=True,
|
|
low_cpu_mem_usage=True,
|
|
)
|
|
|
|
print("✓ Model loaded to CPU with BnB 4-bit")
|
|
|
|
# Count parameters
|
|
total_params = sum(p.numel() for p in model.parameters())
|
|
print(f"\nTotal parameters: {total_params / 1e9:.2f}B")
|
|
|
|
# Check for quantized parameters
|
|
bnb_params = 0
|
|
bf16_params = 0
|
|
for name, p in model.named_parameters():
|
|
if hasattr(p, 'quant_state') and p.quant_state is not None:
|
|
bnb_params += p.numel()
|
|
else:
|
|
bf16_params += p.numel()
|
|
|
|
print(f"BnB 4-bit parameters: {bnb_params / 1e9:.2f}B")
|
|
print(f"BF16 parameters: {bf16_params / 1e9:.2f}B")
|
|
print(f"Estimated size: {(bnb_params * 0.5 + bf16_params * 2) / 1e9:.2f} GB")
|
|
|
|
# Try to move to GPU
|
|
print("\n Moving to GPU 0...")
|
|
try:
|
|
model = model.to("cuda:0")
|
|
print(f"✓ Success! GPU 0: {torch.cuda.memory_allocated(0) / 1e9:.2f} GB")
|
|
print(f" Free VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9 - torch.cuda.memory_allocated(0) / 1e9:.2f} GB")
|
|
except Exception as e:
|
|
print(f"✗ FAILED: {e}")
|
|
|
|
del model
|
|
import gc
|
|
gc.collect()
|
|
torch.cuda.empty_cache()
|