diff --git a/check_model_size.py b/check_model_size.py index 3cb53d6..76c9cab 100644 --- a/check_model_size.py +++ b/check_model_size.py @@ -1,22 +1,34 @@ #!/usr/bin/env python3 -"""Check model size and quantization status.""" +"""Test loading bf16 model with BnB 4-bit to CPU, then report size.""" import torch -from transformers import AutoModelForCausalLM +from transformers import AutoModelForCausalLM, BitsAndBytesConfig -model_path = "/data/models/Ornith-1.0-35B-bnb-4bit" +model_path = "/data/models/Ornith-1.0-35B" print(f"Loading model from: {model_path}") +print(f"\nApplying BnB 4-bit quantization...") + +bnb_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type="nf4", + bnb_4bit_compute_dtype=torch.float16, +) + model = AutoModelForCausalLM.from_pretrained( model_path, + quantization_config=bnb_config, device_map="cpu", torch_dtype=torch.float16, trust_remote_code=True, + low_cpu_mem_usage=True, ) +print("✓ Model loaded to CPU with BnB 4-bit") + # Count parameters total_params = sum(p.numel() for p in model.parameters()) -print(f"Total parameters: {total_params / 1e9:.2f}B") +print(f"\nTotal parameters: {total_params / 1e9:.2f}B") # Check for quantized parameters bnb_params = 0 @@ -31,6 +43,16 @@ print(f"BnB 4-bit parameters: {bnb_params / 1e9:.2f}B") print(f"BF16 parameters: {bf16_params / 1e9:.2f}B") print(f"Estimated size: {(bnb_params * 0.5 + bf16_params * 2) / 1e9:.2f} GB") +# Try to move to GPU +print("\n Moving to GPU 0...") +try: + model = model.to("cuda:0") + print(f"✓ Success! GPU 0: {torch.cuda.memory_allocated(0) / 1e9:.2f} GB") + print(f" Free VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9 - torch.cuda.memory_allocated(0) / 1e9:.2f} GB") +except Exception as e: + print(f"✗ FAILED: {e}") + del model import gc gc.collect() +torch.cuda.empty_cache()