From 410718bc73af83d80a0e252dadbef358b1abf501 Mon Sep 17 00:00:00 2001
From: Christian Medina <37550954+cmedinasoriano@users.noreply.github.com>
Date: Thu, 2 Jul 2026 22:33:36 -0400
Subject: [PATCH] =?UTF-8?q?feat:=20add=20CPU=20offloading=20for=2070GB=20?=
 =?UTF-8?q?=E2=86=92=2017GB=20conversion?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 quantize_model.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/quantize_model.py b/quantize_model.py
index e657d79..e8c6bea 100644
--- a/quantize_model.py
+++ b/quantize_model.py
@@ -16,13 +16,13 @@ def quantize_model(model_path, output_path):
     )
 
     print("Loading model with 4-bit quantization...")
-    print("   This will distribute across both GPUs automatically\n")
+    print("   Using CPU offloading to handle 70GB bf16 → 17GB 4-bit conversion\n")
 
     model = AutoModelForCausalLM.from_pretrained(
         model_path,
         quantization_config=bnb_config,
         device_map="auto",
-        max_memory={0: "28GiB", 1: "28GiB"},  # Leave room for training
+        max_memory={0: "28GiB", 1: "28GiB", "cpu": "120GiB"},
         low_cpu_mem_usage=True,
         trust_remote_code=True,
     )