74 lines
2.5 KiB
Python
74 lines
2.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Streaming NF4 quantization using accelerate device_map."""
|
|
|
|
import argparse
|
|
import gc
|
|
import torch
|
|
from pathlib import Path
|
|
from transformers import AutoModelForCausalLM, BitsAndBytesConfig, AutoConfig
|
|
|
|
|
|
def quantize_weight_nf4(weight: torch.Tensor):
|
|
"""Quantize a single weight tensor to NF4 using Linear4bit."""
|
|
if weight.dim() != 2:
|
|
return weight, None
|
|
|
|
# Create Linear4bit and quantize
|
|
in_features = weight.size(1)
|
|
out_features = weight.size(0)
|
|
linear = Linear4bit(in_features, out_features, bias=False, compute_dtype=torch.float16, quant_type="nf4")
|
|
|
|
with torch.no_grad():
|
|
linear.weight = nn.Parameter(weight.clone())
|
|
# Force quantization
|
|
_ = linear.weight.quant_state
|
|
|
|
# Return quantized weight
|
|
return linear.weight, None
|
|
|
|
|
|
def streaming_quantize(model_path: str, output_path: str):
|
|
print(f"Streaming NF4 quantization: {model_path}")
|
|
output_path = Path(output_path)
|
|
output_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
bnb_config = BitsAndBytesConfig(
|
|
load_in_4bit=True,
|
|
bnb_4bit_quant_type="nf4",
|
|
bnb_4bit_compute_dtype=torch.float16,
|
|
bnb_4bit_use_double_quant=True,
|
|
)
|
|
|
|
print("Loading model with BnB 4-bit (shards streamed to GPUs)...")
|
|
print(" This will distribute across both GPUs\n")
|
|
|
|
model = AutoModelForCausalLM.from_pretrained(
|
|
model_path,
|
|
quantization_config=bnb_config,
|
|
device_map="auto",
|
|
max_memory={0: "28GiB", 1: "28GiB", "cpu": "120GiB"},
|
|
low_cpu_mem_usage=True,
|
|
trust_remote_code=True,
|
|
)
|
|
|
|
# Count quantized parameters
|
|
total_params = sum(p.numel() for p in model.parameters())
|
|
bnb_params = sum(p.numel() for p in model.parameters() if hasattr(p, 'quant_state') and p.quant_state is not None)
|
|
print(f"✓ Model loaded and quantized")
|
|
print(f" Total: {total_params / 1e9:.2f}B parameters")
|
|
print(f" Quantized: {bnb_params / 1e9:.2f}B parameters ({bnb_params/total_params*100:.1f}%)\n")
|
|
|
|
print(f"Saving quantized model to: {output_path}")
|
|
model.save_pretrained(output_path)
|
|
|
|
print(f"\n✅ Quantized model saved to: {output_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--model-path", type=str, default="/data/models/Ornith-1.0-35B")
|
|
parser.add_argument("--output-path", type=str, default="/data/models/Ornith-1.0-35B-nf4")
|
|
args = parser.parse_args()
|
|
|
|
streaming_quantize(args.model_path, args.output_path)
|