diff --git a/quantize_streaming.py b/quantize_streaming.py index 3afd486..0f5ef02 100644 --- a/quantize_streaming.py +++ b/quantize_streaming.py @@ -39,9 +39,10 @@ def streaming_quantize(model_path: str, output_path: str): bnb_4bit_use_double_quant=True, ) - print("Loading model with BnB 4-bit (shards streamed to GPUs)...") - print(" This will distribute across both GPUs\n") + print("Loading model with BnB 4-bit (shards streamed via accelerate)...") + print(" This will stream shards and quantize them\n") + # This streams shards and applies BnB correctly model = AutoModelForCausalLM.from_pretrained( model_path, quantization_config=bnb_config,