56 lines
1.8 KiB
Python
56 lines
1.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Simple NF4 quantization using BnB with device_map auto-distribution."""
|
|
|
|
import torch
|
|
from transformers import AutoModelForCausalLM, BitsAndBytesConfig, AutoTokenizer
|
|
|
|
|
|
def quantize_model(model_path, output_path):
|
|
print(f"Quantizing model from: {model_path}")
|
|
|
|
bnb_config = BitsAndBytesConfig(
|
|
load_in_4bit=True,
|
|
bnb_4bit_quant_type="nf4",
|
|
bnb_4bit_compute_dtype=torch.bfloat16,
|
|
bnb_4bit_use_double_quant=True,
|
|
)
|
|
|
|
print("Loading model with 4-bit quantization...")
|
|
print(" This will distribute across both GPUs automatically\n")
|
|
|
|
model = AutoModelForCausalLM.from_pretrained(
|
|
model_path,
|
|
quantization_config=bnb_config,
|
|
device_map="auto",
|
|
max_memory={0: "28GiB", 1: "28GiB"}, # Leave room for training
|
|
low_cpu_mem_usage=True,
|
|
trust_remote_code=True,
|
|
)
|
|
|
|
print("\n✓ Model loaded and quantized")
|
|
print(f" GPU 0: {torch.cuda.memory_allocated(0) / 1e9:.2f} GB")
|
|
print(f" GPU 1: {torch.cuda.memory_allocated(1) / 1e9:.2f} GB")
|
|
|
|
print(f"\nSaving quantized model to: {output_path}")
|
|
model.save_pretrained(output_path)
|
|
|
|
# Save tokenizer
|
|
try:
|
|
tokenizer = AutoTokenizer.from_pretrained(model_path)
|
|
tokenizer.save_pretrained(output_path)
|
|
print("✓ Tokenizer saved")
|
|
except:
|
|
print("⚠ No tokenizer found")
|
|
|
|
print(f"\n✅ Quantized model saved to: {output_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--model-path", type=str, default="/data/models/Ornith-1.0-35B")
|
|
parser.add_argument("--output-path", type=str, default="/data/models/Ornith-1.0-35B-nf4")
|
|
args = parser.parse_args()
|
|
|
|
quantize_model(args.model_path, args.output_path)
|