diff --git a/test_model_loading.py b/test_model_loading.py index 122018c..72859e8 100644 --- a/test_model_loading.py +++ b/test_model_loading.py @@ -31,9 +31,9 @@ def test_model_loading(): print(" Loading model...") model = AutoModelForCausalLM.from_pretrained( - "/data/models/Ornith-1.0-35B", - quantization_config=bnb_config, + "/data/models/Ornith-1.0-35B-4bit", # Use already-quantized 4-bit model device_map="auto", + torch_dtype=torch.float16, trust_remote_code=True, low_cpu_mem_usage=True, )