From 8a30ec92c6ae79b31bd3f986cb9acc8604db7c2a Mon Sep 17 00:00:00 2001
From: Christian Medina <37550954+cmedinasoriano@users.noreply.github.com>
Date: Fri, 3 Jul 2026 00:23:21 -0400
Subject: [PATCH] fix: reduce to 3 shards per GPU (6 total) for memory

---
 quantize_streaming.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/quantize_streaming.py b/quantize_streaming.py
index 5aecf2d..4537071 100644
--- a/quantize_streaming.py
+++ b/quantize_streaming.py
@@ -47,8 +47,8 @@ def streaming_quantize(model_path: str, output_path: str):
     
     config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
 
-    # Process 4 shards per GPU (8 total) for max parallelism
-    shards_per_gpu = 4
+    # Process 3 shards per GPU (6 total) - adjust based on GPU memory
+    shards_per_gpu = 3
     num_gpus = 2
     max_parallel = shards_per_gpu * num_gpus