diff --git a/train-on-this-server.sh b/train-on-this-server.sh index 815d187..fd9b0b1 100755 --- a/train-on-this-server.sh +++ b/train-on-this-server.sh @@ -80,7 +80,17 @@ echo "" # Use torchrun for distributed training (2 GPUs) # Add --check-only to validate without training: # torchrun --nproc_per_node=2 training/scripts/train.py --config training/configs/ornith-35b-lora.yaml --check-only -torchrun --nproc_per_node=2 training/scripts/train.py --config training/configs/ornith-35b-lora.yaml + +# Run in screen if available (bypasses cgroup limits) +if command -v screen &> /dev/null; then + echo "Running in screen session..." + screen -dmS lora_training bash -c "torchrun --nproc_per_node=2 training/scripts/train.py --config training/configs/ornith-35b-lora.yaml" + echo "Training started in screen session 'lora_training'" + echo "Watch with: screen -r lora_training" +else + echo "Screen not available, running directly..." + torchrun --nproc_per_node=2 training/scripts/train.py --config training/configs/ornith-35b-lora.yaml +fi echo "" echo "=== Training Complete ==="