diff --git a/training/scripts/train.py b/training/scripts/train.py index 947c61b..423afd3 100755 --- a/training/scripts/train.py +++ b/training/scripts/train.py @@ -59,8 +59,8 @@ def train(config_path): except Exception as e: print(f"✗ Failed: {e}") - # Strategy 1: Load 4-bit model AS-IS (no quantization, no DeepSpeed) - print("\n[1/2] Trying: 4-bit model AS-IS...") + # Strategy 1: 4-bit AS-IS with auto device_map + print("\n[1/6] Trying: 4-bit AS-IS...") try: model = AutoModelForCausalLM.from_pretrained( config["base_model"], @@ -68,24 +68,52 @@ def train(config_path): device_map="auto", trust_remote_code=True, ) - print("✓ Success: 4-bit model loaded") + print("✓ Success: 4-bit AS-IS") except Exception as e: print(f"✗ Failed: {e}") - # Strategy 2: bf16 to CPU - print("\n[2/2] Trying: bf16 model to CPU...") + # Strategy 2: 4-bit with bf16 compute + print("\n[2/6] Trying: 4-bit with bf16 compute...") try: - model = AutoModelForCausalLM.from_pretrained( - config["base_model"], - torch_dtype=torch.bfloat16, - device_map="cpu", - low_cpu_mem_usage=True, - trust_remote_code=True, - ) - print("✓ Success: bf16 model loaded to CPU") + from transformers import BitsAndBytesConfig + bnb_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True) + model = AutoModelForCausalLM.from_pretrained(config["base_model"], quantization_config=bnb_config, device_map="auto", trust_remote_code=True) + print("✓ Success: 4-bit with bf16 compute") except Exception as e: print(f"✗ Failed: {e}") - raise RuntimeError("All loading strategies failed!") + + # Strategy 3: 4-bit to CPU + print("\n[3/6] Trying: 4-bit to CPU...") + try: + model = AutoModelForCausalLM.from_pretrained(config["base_model"], torch_dtype=torch.float16, device_map="cpu", trust_remote_code=True) + print("✓ Success: 4-bit to CPU") + except Exception as e: + print(f"✗ Failed: {e}") + + # Strategy 4: 4-bit fp32 + print("\n[4/6] Trying: 4-bit fp32...") + try: + model = AutoModelForCausalLM.from_pretrained(config["base_model"], torch_dtype=torch.float32, device_map="auto", trust_remote_code=True) + print("✓ Success: 4-bit fp32") + except Exception as e: + print(f"✗ Failed: {e}") + + # Strategy 5: bf16 to CPU + print("\n[5/6] Trying: bf16 to CPU...") + try: + model = AutoModelForCausalLM.from_pretrained(config["base_model"], torch_dtype=torch.bfloat16, device_map="cpu", low_cpu_mem_usage=True, trust_remote_code=True) + print("✓ Success: bf16 to CPU") + except Exception as e: + print(f"✗ Failed: {e}") + + # Strategy 6: bf16 auto + print("\n[6/6] Trying: bf16 auto...") + try: + model = AutoModelForCausalLM.from_pretrained(config["base_model"], torch_dtype=torch.bfloat16, device_map="auto", low_cpu_mem_usage=True, trust_remote_code=True) + print("✓ Success: bf16 auto") + except Exception as e: + print(f"✗ Failed: {e}") + raise RuntimeError("All loading strategies failed!") # Prepare model for k-bit training from peft import prepare_model_for_kbit_training